From e389bfe568c2ee7586665d9bb44de2e56273f5f6 Mon Sep 17 00:00:00 2001 From: Andy Gee Date: Tue, 29 Jun 2021 16:51:46 -0700 Subject: [PATCH] update sheepdog for postgresql ssl support --- .dockerignore | 4 + Dockerfile | 5 + bin/setup_psqlgraph.py | 151 +++++++-- bin/setup_test_database.py | 49 ++- bin/setup_transactionlogs.py | 28 +- .../postgresql/postgresql_test_init.sql | 1 + deployment/uwsgi/uwsgi.ini | 4 + dev-requirements.txt | 4 +- docs/api_reference/sheepdog.auth.rst | 15 - docs/api_reference/sheepdog.dictionary.rst | 7 - docs/api_reference/sheepdog.models.rst | 7 - docs/api_reference/sheepdog.rst | 2 - docs/api_reference/substitutions.rst | 2 +- docs/azure_devops_pipeline.md | 126 +++++++ docs/conf.py | 10 +- docs/local_dev_environment.md | 219 +++++++++++++ docs/pipeline_config_1.png | Bin 0 -> 6561 bytes docs/pipeline_config_2.png | Bin 0 -> 28706 bytes docs/pipeline_config_3.png | Bin 0 -> 22416 bytes docs/requests/index.rst | 4 +- pipeline.yaml | 147 +++++++++ requirements.txt | 10 +- run_tests.bash | 2 +- sample-usage.py | 14 + sheepdog/api.py | 54 ++- .../blueprint/routes/views/program/project.py | 6 +- sheepdog/test_settings.py | 4 +- sheepdog/utils/transforms/__init__.py | 7 +- sheepdog/utils/transforms/bcr_xml_to_json.py | 46 ++- sheepdog/utils/transforms/graph_to_doc.py | 94 ++++-- tests/integration/datadict/conftest.py | 55 +++- .../datadict/submission/test_endpoints.py | 307 +++++++++++++----- .../datadict/submission/test_upload.py | 291 +++++++++++------ .../integration/datadictwithobjid/conftest.py | 51 ++- .../submission/test_endpoints.py | 55 +++- .../submission/test_upload.py | 242 +++++++++----- 36 files changed, 1561 insertions(+), 462 deletions(-) create mode 100644 .dockerignore create mode 100644 deployment/scripts/postgresql/postgresql_test_init.sql delete mode 100644 docs/api_reference/sheepdog.dictionary.rst delete mode 100644 docs/api_reference/sheepdog.models.rst create mode 100644 docs/azure_devops_pipeline.md create mode 100644 docs/local_dev_environment.md create mode 100644 docs/pipeline_config_1.png create mode 100644 docs/pipeline_config_2.png create mode 100644 docs/pipeline_config_3.png create mode 100644 pipeline.yaml create mode 100644 sample-usage.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..567f59722 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +!.git/** +!.git +!.gitignore +!.github/** diff --git a/Dockerfile b/Dockerfile index 740a136cb..607ec568d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,11 @@ COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini WORKDIR /sheepdog ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 +# TODO: consider pinned version of pip. +# hadolint ignore=DL3013 RUN python -m pip install --upgrade pip \ && python -m pip install --upgrade setuptools \ + && python /sheepdog/setup.py install \ && pip --version \ && pip install -r requirements.txt @@ -24,6 +27,8 @@ RUN mkdir -p /var/www/sheepdog \ EXPOSE 80 +# TODO: Check using legacy notation instead of backticked +# hadolint ignore=SC2006 RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >sheepdog/version_data.py \ && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>sheepdog/version_data.py \ && python setup.py install diff --git a/bin/setup_psqlgraph.py b/bin/setup_psqlgraph.py index 9911541f8..ae2b2c7db 100755 --- a/bin/setup_psqlgraph.py +++ b/bin/setup_psqlgraph.py @@ -7,15 +7,31 @@ from psqlgraph import create_all, Node, Edge -def try_drop_test_data(user, database, root_user="postgres", host="", root_password=""): +def try_drop_test_data( # nosec + user, + database="postgres", + root_user="postgres", + host="", + port="5432", + root_password="", + default_database="postgres", + use_ssl=False, +): print("Dropping old test data") - connect_str = "postgres://{user}@{host}/postgres".format(user=root_user, host=host) - if root_password: - connect_str = "postgres://{user}:{password}@{host}/postgres".format( - user=root_user, password=root_password, host=host - ) + connect_str = _get_connection_string( + user=root_user, + password=root_password, + host=host, + port=port, + database=default_database, + ) + + # added in for Postgresql SSL testing. + connect_args = {} + if use_ssl: + connect_args["sslmode"] = "require" - engine = create_engine(connect_str) + engine = create_engine(connect_str, connect_args=connect_args) conn = engine.connect() conn.execute("commit") @@ -29,15 +45,33 @@ def try_drop_test_data(user, database, root_user="postgres", host="", root_passw conn.close() -def setup_database( +def _get_connection_string(user, password, host, port, database): + connect_str = "postgres://{user}@{host}:{port}/{database}".format( + user=user, host=host, port=port, database=database + ) + if password: + connect_str = "postgres://{user}:{password}@{host}:{port}/{database}".format( + user=user, + password=password, + host=host, + port=port, + database=database, + ) + return connect_str + + +def setup_database( # nosec user, password, database, root_user="postgres", host="", + port="5432", no_drop=False, no_user=False, root_password="", + default_database="postgres", + use_ssl=False, ): """ setup the user and database @@ -45,18 +79,35 @@ def setup_database( print("Setting up test database") if not no_drop: - try_drop_test_data(user, database, root_user, host, root_password) - - connect_str = "postgres://{user}@{host}/postgres".format(user=root_user, host=host) - if password: - connect_str = "postgres://{user}:{password}@{host}/postgres".format( - user=root_user, password=root_password, host=host + try_drop_test_data( + user=user, + database=database, + root_user=root_user, + host=host, + port=port, + root_password=root_password, + default_database=default_database, + use_ssl=use_ssl, ) - engine = create_engine(connect_str) + connect_str = _get_connection_string( + user=root_user, + password=root_password, + host=host, + port=port, + database=default_database, + ) + + # added in for Postgresql SSL testing. + connect_args = {} + if use_ssl: + connect_args["sslmode"] = "require" + + engine = create_engine(connect_str, connect_args=connect_args) conn = engine.connect() conn.execute("commit") + # Use default db connection to set up schema create_stmt = 'CREATE DATABASE "{database}"'.format(database=database) try: conn.execute(create_stmt) @@ -65,8 +116,9 @@ def setup_database( if not no_user: try: + user_no_host = user if "@" not in user else user.split("@")[0] user_stmt = "CREATE USER {user} WITH PASSWORD '{password}'".format( - user=user, password=password + user=user_no_host, password=password ) conn.execute(user_stmt) except Exception as msg: @@ -74,8 +126,8 @@ def setup_database( # User may already exist - GRANT privs on new db try: perm_stmt = ( - "GRANT ALL PRIVILEGES ON DATABASE {database} to {password}" - "".format(database=database, password=password) + "GRANT ALL PRIVILEGES ON DATABASE {database} to {user}" + "".format(database=database, user=user_no_host) ) conn.execute(perm_stmt) conn.execute("commit") @@ -84,27 +136,40 @@ def setup_database( conn.close() -def create_tables(host, user, password, database): +def create_tables(host, port, user, password, database, use_ssl=False): """ create a table """ print("Creating tables in test database") + # added for Postgresql SSL + connect_args = {} + if use_ssl: + connect_args["sslmode"] = "require" + engine = create_engine( - "postgres://{user}:{pwd}@{host}/{db}".format( - user=user, host=host, pwd=password, db=database - ) + _get_connection_string( + user=user, password=password, host=host, port=port, database=database + ), + connect_args=connect_args, ) create_all(engine) versioned_nodes.Base.metadata.create_all(engine) -def create_indexes(host, user, password, database): +def create_indexes(host, port, user, password, database, use_ssl=False): print("Creating indexes") + + # added for Postgresql SSL + connect_args = {} + if use_ssl: + connect_args["sslmode"] = "require" + engine = create_engine( - "postgres://{user}:{pwd}@{host}/{db}".format( - user=user, host=host, pwd=password, db=database - ) + _get_connection_string( + user=user, password=password, host=host, port=port, database=database + ), + connect_args=connect_args, ) index = lambda t, c: ["CREATE INDEX ON {} ({})".format(t, x) for x in c] for scls in Node.get_subclasses(): @@ -135,6 +200,9 @@ def create_indexes(host, user, password, database): parser.add_argument( "--host", type=str, action="store", default="localhost", help="psql-server host" ) + parser.add_argument( + "--port", type=str, action="store", default="5432", help="psql-server port" + ) parser.add_argument( "--user", type=str, action="store", default="test", help="psql test user" ) @@ -152,20 +220,47 @@ def create_indexes(host, user, password, database): default="sheepdog_automated_test", help="psql test database", ) + parser.add_argument( + "--default-database", + type=str, + action="store", + default="postgres", + help="psql test database for root user", + ) parser.add_argument( "--no-drop", action="store_true", default=False, help="do not drop any data" ) parser.add_argument( "--no-user", action="store_true", default=False, help="do not create user" ) + parser.add_argument( + "--use-ssl", type=bool, action="store", default=False, help="Use Psql SSL" + ) args = parser.parse_args() setup_database( args.user, args.password, args.database, + port=args.port, no_drop=args.no_drop, no_user=args.no_user, + default_database=args.default_database, + use_ssl=args.use_ssl, + ) + create_tables( + args.host, + args.port, + args.user, + args.password, + args.database, + use_ssl=args.use_ssl, + ) + create_indexes( + args.host, + args.port, + args.user, + args.password, + args.database, + use_ssl=args.use_ssl, ) - create_tables(args.host, args.user, args.password, args.database) - create_indexes(args.host, args.user, args.password, args.database) diff --git a/bin/setup_test_database.py b/bin/setup_test_database.py index 07b58e530..8be8711e2 100644 --- a/bin/setup_test_database.py +++ b/bin/setup_test_database.py @@ -19,6 +19,9 @@ parser.add_argument( "--host", type=str, action="store", default="localhost", help="psql-server host" ) + parser.add_argument( + "--port", type=str, action="store", default="5432", help="psql-server port" + ) parser.add_argument( "--user", type=str, action="store", default="test", help="psql test user" ) @@ -29,6 +32,13 @@ default="test", help="psql test password", ) + parser.add_argument( + "--root_user", + type=str, + action="store", + default="postgres", + help="psql root (postgres) user name", + ) parser.add_argument( "--root_password", type=str, @@ -49,17 +59,44 @@ parser.add_argument( "--no-user", action="store_true", default=False, help="do not create user" ) + parser.add_argument( + "--use-ssl", type=bool, action="store", default=False, help="Use Psql SSL" + ) args = parser.parse_args() setup_database( - args.user, - args.password, - args.database, + user=args.user, + password=args.password, + database=args.database, + root_user=args.root_user, host=args.host, + port=args.port, root_password=args.root_password, no_drop=args.no_drop, no_user=args.no_user, + use_ssl=args.use_ssl, + ) + create_tables( + args.host, + args.port, + args.user, + args.password, + args.database, + use_ssl=args.use_ssl, + ) + create_indexes( + args.host, + args.port, + args.user, + args.password, + args.database, + use_ssl=args.use_ssl, + ) + create_transaction_logs_table( + args.host, + args.port, + args.user, + args.password, + args.database, + use_ssl=args.use_ssl, ) - create_tables(args.host, args.user, args.password, args.database) - create_indexes(args.host, args.user, args.password, args.database) - create_transaction_logs_table(args.host, args.user, args.password, args.database) diff --git a/bin/setup_transactionlogs.py b/bin/setup_transactionlogs.py index d6dc233f8..d395c9117 100644 --- a/bin/setup_transactionlogs.py +++ b/bin/setup_transactionlogs.py @@ -8,11 +8,16 @@ from gdcdatamodel.models.submission import Base -def setup(host, user, password, database): +def setup(host, port, user, password, database, use_ssl=False): + connect_args = {} + if use_ssl: + connect_args["sslmode"] = "require" + engine = create_engine( - "postgres://{user}:{password}@{host}/{database}".format( - user=user, host=host, password=password, database=database - ) + "postgres://{user}:{password}@{host}:{port}/{database}".format( + user=user, host=host, port=port, password=password, database=database + ), + connect_args=connect_args, ) Base.metadata.create_all(engine) @@ -23,6 +28,9 @@ def setup(host, user, password, database): parser.add_argument( "--host", type=str, action="store", default="localhost", help="psql-server host" ) + parser.add_argument( + "--port", type=str, action="store", default="5432", help="psql-server port" + ) parser.add_argument( "--user", type=str, action="store", default="test", help="psql test user" ) @@ -40,6 +48,16 @@ def setup(host, user, password, database): default="sheepdog_automated_test", help="psql test database", ) + parser.add_argument( + "--use-ssl", type=bool, action="store", default=False, help="Use Psql SSL" + ) args = parser.parse_args() - setup(args.host, args.user, args.password, args.database) + setup( + args.host, + args.port, + args.user, + args.password, + args.database, + use_ssl=args.use_ssl, + ) diff --git a/deployment/scripts/postgresql/postgresql_test_init.sql b/deployment/scripts/postgresql/postgresql_test_init.sql new file mode 100644 index 000000000..b49dad2da --- /dev/null +++ b/deployment/scripts/postgresql/postgresql_test_init.sql @@ -0,0 +1 @@ +CREATE DATABASE sheepdog_automated_test; diff --git a/deployment/uwsgi/uwsgi.ini b/deployment/uwsgi/uwsgi.ini index 38ec24529..f1a4731f3 100644 --- a/deployment/uwsgi/uwsgi.ini +++ b/deployment/uwsgi/uwsgi.ini @@ -11,6 +11,10 @@ harakiri-verbose = true # No global HARAKIRI, using only user HARAKIRI, because export overwrites it # Cannot overwrite global HARAKIRI with user's: https://git.io/fjYuD # harakiri = 45 +; If VIRTUAL_ENV is set then use its value to specify the virtualenv directory +if-env = VIRTUAL_ENV +virtualenv = %(_) +endif = http-timeout = 45 socket-timeout = 45 worker-reload-mercy = 45 diff --git a/dev-requirements.txt b/dev-requirements.txt index 541886939..f429e8cc7 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,13 +3,13 @@ pytest-cov==2.5.1 requests_mock==1.4.0 httmock==1.2.3 lockfile==0.12.2 -coverage==4.0.0 +coverage==5.3.0 mock==1.0.1 pytest-flask==0.15.0 moto==0.4.5 sphinxcontrib-httpdomain==1.3.0 -codacy-coverage +codacy-coverage==1.3.11 Sphinx==1.6.5 sphinx_rtd_theme flasgger==0.9.1 diff --git a/docs/api_reference/sheepdog.auth.rst b/docs/api_reference/sheepdog.auth.rst index 9cbae8afd..10d01a7db 100644 --- a/docs/api_reference/sheepdog.auth.rst +++ b/docs/api_reference/sheepdog.auth.rst @@ -6,18 +6,3 @@ :undoc-members: :show-inheritance: -``sheepdog.auth.auth_driver`` ------------------------------ - -.. automodule:: sheepdog.auth.auth_driver - :members: - :undoc-members: - :show-inheritance: - -``sheepdog.auth.federated_user`` --------------------------------- - -.. automodule:: sheepdog.auth.federated_user - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api_reference/sheepdog.dictionary.rst b/docs/api_reference/sheepdog.dictionary.rst deleted file mode 100644 index d9def4794..000000000 --- a/docs/api_reference/sheepdog.dictionary.rst +++ /dev/null @@ -1,7 +0,0 @@ -``sheepdog.dictionary`` -======================= - -.. automodule:: sheepdog.dictionary - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api_reference/sheepdog.models.rst b/docs/api_reference/sheepdog.models.rst deleted file mode 100644 index 945c5f216..000000000 --- a/docs/api_reference/sheepdog.models.rst +++ /dev/null @@ -1,7 +0,0 @@ -``sheepdog.models`` -=================== - -.. automodule:: sheepdog.models - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api_reference/sheepdog.rst b/docs/api_reference/sheepdog.rst index 27a4092c0..531e1777e 100644 --- a/docs/api_reference/sheepdog.rst +++ b/docs/api_reference/sheepdog.rst @@ -2,10 +2,8 @@ sheepdog.auth sheepdog.blueprint - sheepdog.dictionary sheepdog.errors sheepdog.globals - sheepdog.models sheepdog.transactions sheepdog.utils diff --git a/docs/api_reference/substitutions.rst b/docs/api_reference/substitutions.rst index e1033ce49..3581d93cc 100644 --- a/docs/api_reference/substitutions.rst +++ b/docs/api_reference/substitutions.rst @@ -21,4 +21,4 @@ .. |resheader_Content-Type| replace:: Will be ``application/json`` or ``application/xml`` depending on - :mailheader:`Accept` header. \ No newline at end of file + :mailheader: `Accept` header. \ No newline at end of file diff --git a/docs/azure_devops_pipeline.md b/docs/azure_devops_pipeline.md new file mode 100644 index 000000000..9265ecdf9 --- /dev/null +++ b/docs/azure_devops_pipeline.md @@ -0,0 +1,126 @@ +# Azure DevOps Build Pipeline + +The purpose of this [Azure DevOps Pipeline](../pipeline.yaml) is to build `sheepdog`, run a test suite, and then push the `sheepdog` container into an [Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-portal). + +## Getting Started + +If you don't already have access, you can use the free sign up with [Azure Devops](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started/pipelines-sign-up?view=azure-devops). + +You can also import the [pipeline](../pipeline.yaml), see these [doc notes](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started/clone-import-pipeline?view=azure-devops&tabs=yaml#export-and-import-a-pipeline) as a guide. + +### Setup Azure Container Registry + +[Create a Service Principal](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli#password-based-authentication) in your Azure Subscription using [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli). + +First, log into `az` cli: + +```bash +az login +az account set -s +``` + +You can create a **service principal** in Azure AD: + +```bash +spObject=$(az ad sp create-for-rbac --name ServicePrincipalName) + +# this can be used for the SP_CLIENT_ID +spClientId=$(echo $spObject | jq -r ".appId") + +# this can be used for the SP_CLIENT_PASSWORD +spPassword=$(echo $spObject | jq -r ".password") + +# this can be used for the TENANT_ID +spTenantId=$(echo $spObject | jq -r ".tenant") +``` + +> You will need to have appropriate permissions in the AAD directory. If you don't have access, please work with your Azure Subscription administrator to obtain a Service Principal. + +You can also create an **Azure Container Registry** using [azure cli](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-azure-cli) or the [portal](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-portal). + +You can use the following `az` cli commands in `bash` for reference: + +```bash +az group create --name myResourceGroup --location eastus +az acr create --resource-group myResourceGroup --name myContainerRegistry --sku Basic +``` + +Also, make sure that the **Service Principal** has rights to the [Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-roles?tabs=azure-cli) to **acrPull** and **acrPush**. + +```bash +acrResourceId="$(az acr show -n myContainerRegistry -g myResourceGroup --query "id" -o tsv)" + +az role assignment create --assignee $spClientId --role 'AcrPull' --scope $acrResourceId + +az role assignment create --assignee $spClientId --role 'AcrPush' --scope $acrResourceId +``` + +To verify if the pipeline context will have access to ACR, you can login. + +> Note, this is an approach for dev / test, but in a production scenario, it is more likely that your SP Credentials used in the Azure DevOps Pipeline would be populated as secrets through variables or Variable Groups. + +```bash +az login --service-principal --username "$spClientId" --password "$spPassword" --tenant "$spTenantId" + +az acr login --name myContainerRegistry +``` + +You can also verify that this service principal will have `ACRPush` and `ACRPull` permission with ACR, which you can check how the [getting started with docker guide](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-docker-cli?tabs=azure-cli) for more details. + +First, pull and tag an image: + +```bash +docker pull mcr.microsoft.com/oss/nginx/nginx:1.15.5-alpine + +docker tag mcr.microsoft.com/oss/nginx/nginx:1.15.5-alpine mycontainerregistry.azurecr.io/samples/nginx +``` + +> Note that the ACR names will default to **lowercase** for the `fqdn`, so make sure that when you're tagging images to use **lowercase** for the ACR name. + +Check that you can push an image to ACR: + +```bash +docker push mycontainerregistry.azurecr.io/samples/nginx +``` + +Check that you can pull an image from ACR: + +```bash +docker pull mycontainerregistry.azurecr.io/samples/nginx +``` + +You can also list out the images in the ACR with `az` cli: + +```bash +az acr repository list --name mycontainerregistry +``` + +## Configuring the Pipeline + +You can set the variables on your **Azure DevOps pipeline**. + +First, make sure you have already [imported your Azure DevOps Pipeline](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started/clone-import-pipeline?view=azure-devops&tabs=yaml#export-and-import-a-pipeline). + +Click on the pipeline and then click edit, which will let you update the variables in the Azure DevOps pipeline: + +![Click on Variables](pipeline_config_1.png) + +Variable Name | Description +------ | ------ +SP_CLIENT_ID | This is your Service Principal Client ID. +SP_CLIENT_PASS | This is your Service Principal Password. You can override this value when running the Azure DevOps pipeline. +TENANT_ID | This is the Azure AD tenant ID where the SP and the ACR reside. +ACR_NAME | This is the Azure Container Registry name. Note, it is not the **FQDN** (e.g. `myacrname` instead of `myacrname.azurecr.io`). +LOCAL_TEST_POSTGRESQL_USERNAME | Automated Test Username for the local `PostgreSQL` in the pipeline for running the `sheepdog` test suite. Default value is `test`. +LOCAL_TEST_POSTGRESQL_PASSWORD | Automated Test Password for the local `PostgreSQL` in the pipeline for running the `sheepdog` test suite. Default value is `test`. +LOCAL_POSTGRESQL_USERNAME | Username in `PostgreSQL`, and used to setup PostgreSQL schema in the pipeline for running the `sheepdog` test suite. Default value is `postgres`. +LOCAL_POSTGRESQL_PASSWORD | Password for the user in `PostgreSQL` used to setup PostgreSQL schema in the pipeline for running the `sheepdog` test suite. Default value is `test`. +LOCAL_POSTGRESQL_PORT | This is the Local PostgreSQL Port number. The default port for a `PostgreSQL` server is `5432`, but you can change this to another port in case this port is already in use on the host. For example you can use `5433`. + +After updating the variables, be sure to click **save**: + +![Save updated variables](pipeline_config_2.png) + +You can run the pipeline to validate the `sheepdog` build and push to ACR. + +![Run the pipeline](pipeline_config_3.png) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 58ca7add0..a216a59dd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,9 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys -import os -import shlex +import sys # pylint: disable=W0611 +import os # pylint: disable=W0611 +import shlex # pylint: disable=W0611 # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -54,7 +54,7 @@ # General information about the project. project = "sheepdog" -copyright = "2017, Center for Data Intensive Science" +copyright = "2017, Center for Data Intensive Science" # pylint: disable=W0622 author = "Center for Data Intensive Science" # The version info for the project you're documenting, acts as replacement for @@ -148,7 +148,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +# html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/docs/local_dev_environment.md b/docs/local_dev_environment.md new file mode 100644 index 000000000..73128b36a --- /dev/null +++ b/docs/local_dev_environment.md @@ -0,0 +1,219 @@ +# Set up a local development environment + +This guide will cover setting up a sheepdog development environment. + +The cloud settings are still to be determined. + +## TODO: Cloud Settings. + +* Google (storage / service accounts) +* AWS (storage / auth) +* Azure (to be determined) + +## Set up Working Directory + +Clone the repo locally. + +```console +git clone https://github.com/uc-cdis/sheepdog.git +``` + +Navigate to the cloned repository directory. + +## Set up Python 3 + +The environment was tested with python 3.8 on WSL1. You can use `bash` to install python 3 if it's not already available. + +```console +sudo apt-get update +sudo apt-get install python3 +``` + +### Set up a Virtual Environment + +Set up a virtual environment for use with this project using `bash`: + +```console +python3 -m venv py3-venv +. py3-venv/bin/activate +``` + +## Set up local Postgresql DB for testing + +You can use a local postgresql for testing purposes. + +### Set up local Postgresql DB on WSL + +You can use `bash` to install postgres: + +```console +sudo apt install postgresql-client-common +sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +sudo apt-get update +sudo apt-get install postgresql-12 +``` + +Make sure the cluster is started: + +```console +sudo pg_ctlcluster 12 main start +``` + +### Set up local Postgresql DB on Mac + +If you're on mac, you can install postgres using brew: + +```console +brew install postgres +``` + +### Set up DB and users for testing + +You'll need to connect to the postgresql and add test users and databases. + +#### Connect to Postgresql on WSL + +Connect to the local postgresql server + +```console +sudo -i -u postgres +psql +``` + +#### Connect to Postgresql on Mac + +If you're on a mac, use the following to connect to postgres: + +```console +brew services start postgres +psql postgres +``` + +#### Helpful psql commands +It may be helpful to understand some psql commands too: + +```console +\conninfo # check connection info +\l # list databases +\d # list tables in database +\c # list short connection info +\c postgres # connect to a database named postgres +\q # quit +``` + +#### Set up users in psql + +Initialize a user within the psql console: + +```console +CREATE USER postgres WITH PASSWORD 'test'; +ALTER USER postgres WITH PASSWORD 'test'; +\du +``` + +## Installation + +### Install for General Usage + +To install sheepdog for use with other Gen3 services, running these commands is sufficient. + +```console +python3 setup.py build +python3 setup.py install +``` + +### Install for Development + +Check the [dev-requirements.txt](https://github.com/uc-cdis/sheepdog/blob/master/dev-requirements.txt) and make sure the requirements includes the following dependencies: + +``` +coverage==5.3.0 +codacy-coverage==1.3.11 +``` + +Install the development requirements with the following commands: + +```console +python3 -m pip install -r dev-requirements.txt +python3 setup.py develop +``` + +Install requirements with pip. + +> One of the requirements `dictionaryutils==3.4.0` may not be available through pip, so you can try with a lower available version. In this case, update [requirements.txt](https://github.com/uc-cdis/sheepdog/blob/master/requirements.txt) to use `dictionaryutils==3.2.0`. You may also need to update the requirement for `gen3authz==0.2.1` to `gen3authz==0.4.0`. + +```console +python3 -m pip install -r requirements.txt +``` + +### Validate Installation + +You can try to run the following to confirm installation: + +```console +python3 sample-usage.py +``` + +For convenience, the minimal usage looks like the following: + +```python +import sheepdog +import datamodelutils +from flask import Flask +from dictionaryutils import dictionary +from gdcdictionary import gdcdictionary +from gdcdatamodel import models, validators + +dictionary.init(gdcdictionary) +datamodelutils.validators.init(validators) +datamodelutils.models.init(models) +blueprint = sheepdog.create_blueprint(name='submission') + +app = Flask(__name__) +app.register_blueprint(blueprint) +``` + +If there's any issues with running the sample, revisit the installation. + +> Note that `import sheepdog` relies on building and installing a local python egg for sheepdog, which can be accomplished using `python setup.py build` and `python setup.py install` in the local root directory for the clone repository. + +### Generate Documentation + +Auto-documentation is set up using [Sphinx](http://www.sphinx-doc.org/en/stable/). To build it, run + +```console +cd docs +make html +``` + +which by default will output the index.html page to docs/build/html/index.html. + +> Note that `make` should be available in the console. For this guide, you can use `make` from WSL1. You may also find that the *.rst files contain outdated definitions, so remove the definitions before building the documentation. + +### Running tests + +Before running the tests, make sure your virtual environment already activated. + +```console +. py3-venv/bin/activate +``` + +For convenience, you can run the tests using [run_tests.bash](https://github.com/uc-cdis/sheepdog/blob/master/run_tests.bash). +> You may need to update the virtual environment activation step in the bash script before running it. You can replace `source ~/virtualenv/python2.7/bin/activate` with `source py3-venv/bin/activate` then run the script with `bash run_tests.bash`. + +You can use the following commands from the working directory for cloned repo in order to set up testing: + +```console +python3 bin/setup_test_database.py +mkdir -p tests/integration/resources/keys; cd tests/integration/resources/keys; openssl genrsa -out test_private_key.pem 2048; openssl rsa -in test_private_key.pem -pubout -out test_public_key.pem; cd - +``` + +Then you can run the following for pytest: + +```console +python3 -m pytest -vv --cov=sheepdog --cov-report xml tests/integration/datadict +python3 -m pytest -vv --cov=sheepdog --cov-report xml --cov-append tests/integration/datadictwithobjid +python3 -m pytest -vv --cov=sheepdog --cov-report xml --cov-append tests/unit +``` + diff --git a/docs/pipeline_config_1.png b/docs/pipeline_config_1.png new file mode 100644 index 0000000000000000000000000000000000000000..c58396ce890509c43e0bc4235b547b8adfbd4a3d GIT binary patch literal 6561 zcmeHLc{r49+t=ceJfRS>H5C$)brMoaNJGe;nQTqgv4k+FrzT|I_wiV=h7`s=31c_O zGWL;SY)Q)XhzT^AzyN~1EuIoC_`@FC7yng5Jyh88mX`MKJ<~S1* z(+M5z+lEX`%!@nE-p3B_T(2@K*zVj8c^YbIFqQTSF6_JC{HVq$OJJrA|P3hbDe_?2{S-+b^4ygU)$EC|i`w7ThgDu|rfkEDvcx}njY?wVjy z8-2K@xc8=@ERRCDhXd%mw$F9G_!Sl|WLukiqu~ zhuY}%AyCA|^tEv=DA(AkSfDw2wk8FN3|M^!d7L2xwGC3Ivr_H7z2zd$_n=fhI$hW? zaOd#gqy4dae?BXF9{PHAfYDKi?kr^UDALypIGgV`JFwxgsHRLSO^0jgQQJGv?H#Vl zw5oJfnc4XCXSz|VA>}{Gn#%GQVByL7)`Ms9ySkXB{u0QM^R@u9wZCAopK%>w82Gw_ z67w1Co|Q@JGAw97ut*$kX=d16c@_GR^Q4c->9rGkg64|e55XhO`qEwMG5TFiHp=iD zk%BRVWg-;rdT(C3W%)Jy$nL-!iQ(HO;KnwjfrsNc#zmH@j;xjTc>*!t>kcHb9M{ud zna7e_pT*#ASDO*o)9h;TJv@Be0Vax?cu21ff0vYmKFjhW=_N;3v46MchwQ;My0!9J zx*)Am&>=nt<4Qdqc_|qIomU}J5@JFXi&G|9_AHX>nQEpgHC;`A zu`EYTos2LGXXh`D``y_zC^8mzG;ZNf^-It{DBCTYPutBh$k+EI&l=&JmPKvK`O(#4 z9_|sG;mVlOE8ZbBP`bJ^n0mEG7Y5lFZbB-~S}Chwi2MSDK%D(GyLWniB75S|F`;gb zN$yuG1jhJOF4P2-Zfke)H~1$A!hl13|9!9|#oz66=B!e3e;N?yB)9GRWIr2#h#2|7J33|K=9Uu>CswQdS%Jli4$ zXuR0cpaItj*%)}4dGfP(ix#5VmU`E9qkQ??$Vv=BXvnq7I5u6#$!X$Fa(pDa6*hl=c!>TR3ZDGXLaRiE(sT)ZFJpl zN(Vs|8PWIzc7hH2Gsok@m|r2gOeq^{pHvGLv0g=&!s18X=;%cR_=rBWa2zCkO#Q?x(jr7ympp3Y|vn@fatMQ!r z$K16;?g9eOy@G^@uo(baguUvduH(6;Su{59hXfRzSmvG%1Z=mOI1&XnN_A5dXCak{ z*m1$FelPz0C}VfoKAKeh- zwJ9OMJ%E;Sc$cNkAU z+Fv^L>q-B%ke?AZNmaVloyTIeSdW*WnpJ|*5Bi`JPnNKOoq#nDd zodMU(uX^MWuSbbv@!2bmE>WAa0r zCdHe>y5;lzu!3WzhHgH{HbO_8zrX!^OPMT3-B$Uw&L5Ue#!2Q5#FE4!t*TkPz0|b1 zKV;;aQNv>2x?KKfk?>}GIfl$5Mgl=S-&$ZiO#?e#B(7^r-^)Q?z(3O>RHkjoToCrY zy>h1OOX|72w zMY{ZF95WbplTwrXj{vEH6V|g^Pg_X|8qMju6sQ)i42}z^doDT$j5Hfl#IaA&BZidw z$TlNGMxw>-!P8i1SEN@_T@}3Fu*L=djcAmSk-sRp7H6$3{Sgq1hiZ5wk^H zI>m2D>KUOg`yw|p6s2L_-U@V}07?XC`ej2y6Dbbcz^1BP!&V&a>_kSOQSyGc3lB%q zha33k(pox(Ps*kLt<_2~ivXV;Q`4=A5x4nsmDAZtHYs@3f5AO>`V{5zN;&(E9#c0(QVrAcb zx2)66VX3jOQFdqTt&8`&IJ}q)Wn>Djy%oX6Pb#w7qV>Ozg_A&^AAD~ot%xD56!7&y zrKW0wp6efI(^Ikyxy=T!6ytQ&xBoPP6d`*HA46~izm19;b@ypcXauCXN<;DC6aJHWa_n(+MmL)he;iw9xfHWCVwGV>#q}xU`0IXy)xN&-Y^zY-E}*CSES#KJ z_b-s~BnmjuSE7$t_Q+(P@z>OYeC@*Kg=vcIn3d+FSM0Y&@hgt3P(!Y)&aC_?=%hS2 zGVz2#?op-XZ496HMb`Ih(}`8UIu+zyu{1K=w$!~BPJ$^4O%zUx3@W zu{PRUhju4j%^8yr5J(8pou*(m=|8iZO~e*TW!&RowNcCWZu2kQD=?>txg&%CdM2q@ zbLgYD@7~)(IyJqCLk3t) z(-|~kf?FzHHZ!&Ib6f&ZKCRR7%-E=&l1MRSW01w=TBfuln}W|%Z}0m1m1@|J))3uI zdF9A?A*UL{7{hV3Y*4NfCb{4|YxS``3saDyE6xO+@pO%%k!OVC} z;UN-Hp(`rhd02}fi3ZyfadR$pl1LaBH5*%lLBfUx*fTO?JjA@KhpIQ4vL+>rU1kP# zo6*`13N=Zv|B#sa#GOL{IXX*n zvZJXl=;*&$M0QsqpImCiMS|OpfkInRxnGNRC>JgBoV6bC?pT8$`G;Bud9OSxqpEWX z{n?h^hRNrL_XSBl z{LR&F5z0D?{q4|gCW5Dt6&F3_ccgY0A3K1Z)%{)wJ7BV3XQpcPJb}YzOtbJ~-ln4E z#g3l7Akp6n9y#RAv{-snn1i=s!pB{TITqyW(A5e-GA*Y8)MEzcTUK5jTX+y_8Gd3T zxw5IsDn7kEW0$t&FfLagfXNM|dsg=)pi%W?4k}-knCJ&7#yeh{I?jy&els-hxp%iZ zKP+-i8vs-b1s;!Sc7$HHF210wLeF236OuC#Z{XMifc8=WE)ROPZw6tNU3utI+g~1J zn)Xack+`CUs#wdB#iIpw?bSSllhdw%MyVFutMx|C1dUC@5D1(5gN*3G0@s0Jm%uwA z4DRVUjm>+~RlC$~^$4Kz)w(Mjd|#55#ThslNXzQ3%4VN6$tq!owU>gIl>YHQ)s+~=BYnPN-#d`6bBa;Q}Rsy}KkK@mnHQZIL`mIEE#o zNtXA3xu9OA6r{s(_3f+LIpHQ9Ss81cc?r>8muWM@l5}i{Z}dLNuFmLrm=Fi+T2&$) zk!n_tc?pS2V(5T3qZ?vprpSzvnkIQ1MdT>yAfn%&0zqQAptEN8!TuKD3CW!zFsAPN zq1nWHb{T@BfVRjk0WLqIa=rs2iZKza@=N!&TQ?)~%T1I=FO=ar=D8e;?8wpK+Yj!D z0MV_gvgftz$bZ*-s)Qutrfa~GWsFX^TW%kmHm~dPPBXGWN-J%)9X8WmHfZFScRe}> zt+>(mEG;}EmMbWh3zm|}qd`2NwoJ+q`-GS&y8nOOH1u$4tlDS1==uw{%OOzyE zpO5@$hs#vIu(ZpmDBHQih$Xfob2f`9GSm)wJZb2(g+jnw}t?MF`1 z5Sq4IJIthqD5`Jm@ub9>DO`ICOGUpF1+5~wA&yCFApMEcnJ_KyHBdzly4UhPXfHp! zpRSIE*~EA*FAEU|68Kpz)h!JyQzMZD;7^CvPE4ixu7RXBpJu_dj53Yg5+(8D?*`zW zD*L-SX2(tSk#dozlK8CKfzO@3l0Z^Le^Qx6CjMsAQzAuBHa71-mbSCKFeMbtZ6b1A z+-@MBLRMKAQ?v4V`)j=(jDHRLL*u2RgecuRj&aD7xzRt?DLDaV$oTf&6-7}f1X3o9 zEN}^Hh3VSzkZg=3|n!1csn~EdYq8*hAT9<`31bp$Mj3RI$x2b%w;o}>WbB` z=ja8Y2E83NuHjgTNyH(v?C`8XF*P%_*+ta#+JVRWwYc3AWfDl?fvJJvBjEu4VEK(S zzdO@WnN9`It^{77q+KUv4dlF^e0`2E)pa%Z!7ut~kx#J~pO)L85I_8Hmgl0<_yRnN z8=)U;i?JQ)0WveU@@QF(sr>cglzjdJ@A*v|ERe>G5FKoaqz>h5YY-p^mNW10o5MAW zhs(<5JBE#3RANnWQig-Jg&czhjL@MVdDF8YBu+-eM3Ub%g4<)~zY7m?+iX;IHlTGa!Eib|GD z1(OAei-Z*X%lgX-rSHZoC5(Ed!vAP5^nkFzTv zdd?T#aF4K1b>~((B^NnT&pZei^s^Dd+)OB|7%~fWKRWjdo$Pn)_@@}MKE8#bxcrkx z@O~z@Y#__WG}s&D$7|uzk-4)1nP@ez_eSIy##L^mWQZ@%i63u*wp#~q_Zsf=@oP9QyRt?b;WDH$ZDp^_tYimfpKsrNqma{2-0jq z=<#7GiDI{UBA4m^)sLmpd>^y;Yc<973t!4CMT3NT9#r&PSNN)B9W%u)k5jPv!clxH z_j9ce*@{W`ihiSPSwPNMMAN!Zw`-%0Z-B*z6hc)jJge3|C-LD``~|#%uW3V=Rp*d- z?O}n2>y#&u<|AOpx$1kG8a-ayRtSXUfcPKK9m4w93|4L&@G+*M{^SB z!oY(id_(2?FZHFGn^&YyE2%?l!BW|^65Q&pM!{EGRl4N`i*8BSYh{ohA^nZPxKRNH z$10PYEF&Lg$cKK}5p6>ZsrY0)r;5*=8AFhI1QJ$xi;M-IRF?K}LAU$gx4=pFZ$*Ja zMVE6F7yW=lt)ntZJZVpeZpaNF%>< z$(2a|Zl{~CHnP=DbTk7c^0U7YUQAeacWH7Dz3L%c^a?R@fA5ilGaVIV=np5F7oQi7p#tq8a%mN^JU$w;K_lhvtG`!3*+PfZI) z-T-e+?zXA7(K{~~{A54@51X;MX`>E9pP=St*^gs-KF+P?Wj(FA2WpkuE6P<|x*OF`&kksZi{4 z5SMS?ih9xIz;Of9MUv~g)SM-zYSGgwzD3dx^Bar(t(tc12{jVwUjykFwbL72@UOGC zZjw~G38x?`P*vk<-Jtw+!?lQev|1Hg&%2vVU=rk%e`9%RQo0lXXw5zcgxKWoHG1 z1~mOB8X7pS!h}jBIpMq8Ow?SJN!hJ66>7V&Z6aezI`MqamU%YKv#GXf=yd}rqn35I zr)!6woRu`$DW3Hy8Mm`5fxAnPn-1Voy25Fyfg2`viJ zoAefl^d6)J5<>nRbneWZJM-Q0{NLyEsL#pS=j^rHTJL_>yN~}vCAs5AnT|prkmL97 z-Bp1=_K_fv1Js9q0l(1b&v=0U_BpD^-G=0}pPK_02TY|Er6G{~5b7PHgW&pz-92qb zP_&lvcVC0;$0rboZR!2H(rV8PmPW`Sdi|bQFA}4L7uV6`Ybr^%9ySKtI&i9|<}k@# z_1zm?(Yrd@FPmk5od`Fg9}0q|o%j%WHe>RJzPNL%R`ST@po{j3Vi=o`9O3uG!q0R& z3_Kiqm&W2SH}$sSvf{?X<+QA=wQ782d%PnqYA`w;RVd(|K+E$1wd~&Rn&|GbY-t)* zo;sDFyl9gQ0p}^o9s|!{ERM>AH>Ee@my)6LOClMGGHSOb&|O3Ia9s#ZGw$lHZ`qGURUPl;rYn)E-}&T+Y2hIPmJg- znFxXlVSO%frzvq~JCVG-(4U)G#N@#C>0RMDa-*Eo>YTDydDLv_Ic@r{ zcwb1!Oc;-n*i?4Up1%jWmAxh-demYzws>-v)@!R4yA_1Zn=M)|UEY^59RORc{9Z%t z{A{ZE^e_QUbRpvsNrhe}b(FqA6D!cn2xy*>2DzV1fDM1)_u6hD&%;QWiG=h-Lc2?~ zY7crN+ZGin+;V83>KVQQYmEe5tUt3l)q=}9;)q4OBlIApret;S-x)0wQMRhP4dgAh zW!Db(#UAC|NoA>x9#t#}=K{ycK?#0dTYk2PwOEZJRpCz>CGqs``=3`k-QO&^gUz~v zZ)*y>svnMCZRX~h!tlHDVEOgSoz|w?61Srh^K0LRcrF^ttTS#`yDS@&CbbYOaExAE z%muOx4K}wziE_c`w!ivGO&TyRvJaJVGi$^WVZ@>StSDw}4u;aNrM5Sro5s*IIs-f# z4xLAPE^?QSj&E60hBAsC;W zCgvE9*(?tG#*VL!aBb2Mwr3Lg`HEwR-QC@=tr|uQ>_owN1Zksx1jb#ED1kW?fFZB2 zkyl{EVpyTeLSMhH=XYjCU#dhb7z>tKu;(abe~V{c+? zGZ#%`yvWPsDi*|g3LBg`&$F5^pIM2ey7sF|a|ru-8#AZOSMP4*4&!OpqP(`FN~rMNpBy_Rk=>AM z$7K5W`S^l=SW)(*NRS87Yiw@g7r9vH@{5AHJ62L$yuxL9RF2KFhva}6NM?RmiRCA? ze~2MYXP1r=<~7k4E7rEPwfx=2MWt~-gCvpq{%!tO2wvxsNy=7Zs0 zChsmQplq=Mi{gGPxZ%LT61)4P^o9lEB)mqZL^O>llXEo*E+e{sPGg3PiYf zOT>H?vFspG?u1BgzDrFesw*l(Qx;6yh zl->Lyh?F%VrDbYk)1GbYiC@Mp?*_j)LcVtv(q~eG=r9h*g1WALAYM^k!J2&B<-%$? zm7}+?nmA3b2-`(s04dI>4oo7EBO>*j2)=w7t8^9I@F4j++pagukPD~cY;&3fd1nn* zMV=*#9EAwK?P8mZq!|`?2v*IDnRH|V0r)6a_m#m8L?0;Aon%xTStd~i28tmweC{F-6S zi7eH`k!30S@h{@bmB!Eql=a|H@=zbA+w3(hkeDyB>Vm^;7ea{w_@$XpIaX~0gA$`u zOlh-7^~!PlaO~BDXXC@z-4f!Jx1I`xY>#)p(e8dSfK_{JgZY7*Ib)1Wh;5#M(~dLk*25)jU4u6~$-AUdn~?$;s|P~(B^lAO^+d;y z)7AMK9f?q~0(xBm$6Oym8VDg#oGE%el^?}+(gs^CDJiMH)Aq>S2hWHURBRsKuWPe4cOFe0{`her#?1fodVVu> zXI75UcIR6U)+3QtYU`WvxD7!*UNs{o?y@q;;F^kPWRz%&7@`0h0MIz};qdD_SgRiJ zs&8>D^Ka*0i{WiY%7vg%@?v2JVh4bi?|#sobY;)^+0K-`sLTZMmEIOXpvWR#Ya-~a zK(raTXXX3sOG$l|_|WOy)z#G~^v=^3LN`(whH{t!et?nRATBz3djm$^=!bcYR9`VW z2RSA?7Nvq?)-Ob6W}MN-VJltNxMADPQPu0Imi;;DlT|DA%w!Upj0Y=V?hNEm%A^7h zK8SC)+ow)8ia67eNM1CBTH3C`&w-F#y$OspKqK@UAlD#u25u-@r>TS--fQI;sKaEV z0)N#9@1w*EpB%8GTMOV_Nv#Suz`LZ7mej8~@KI4w=&d?_XR=~@Yilb26!5Q~>OBSu z`!h;Dm6UstcQZA0clb{Zj`zJMsyhw(#C-7ZVi{k( z5@b7DHm-{$^!4CU; z?g}>T8WKJ9OBvs0TgR?BAFNvMwMen5{4j^GmAuI1I0O1x@ozaTy)TX{o0ozCUTxuD zPG4Ihm@E}hc+q;U3BR(6=6lBToF-Cn=DemL0Bnd!QuQ>%+0X1${bBp&l zFFN1>Pnn$)>YPbFsc`FvF}o?h9=*o?BCWo(G7Nt~mm(*kPR(xP@q6AaREKA~^#mn8 zXAUy0yp$B4tCTrNAaf`Y=L5>8egTU8g1 z^}T81y^V5a>2J60xB|}?amgs}wUakz9t*#SuU%2Hr?Rj;_~w^XeWg@PqXW9B5z#S` zbFb7;-vN6YOiAo^yUXru$nLDB*M=q@H)3h40Nrj<9>Q0JT~!{fbXC3MZg|$TbNO~( zT_THbzz+Np3$UGDI1<`3CI6eSO^NJNRw>8TA(n_Q+pKW6%e6o70A& zfkOIqRnAL#98tfBVeWT_t6+zEGc?tnde>I$yL>x3O4SErD~|U&Lx|KaEG!gj^zN!@ zzt~}*yXk{I;I+}Kseq-`yIa0uG?M>Wc*G~2b5~E~7^cgpQO07<{lWDCra^b>(z0RM zP7d&LdZOu;#)#5sO_m%Ja#q#;I`Ws#G=b2?gI&q3)S<-e>0LYQT*bS>6HWnJj(Qqm zGV(gD+Dtli>^k(qQ`MZZ4BfTob66t>!n+Vz;~I`$Mm?#H?|RaT!iG3=a;-<>kK!Nd zWN*^YTl!*8KmWk7`HM_UC~+j~0-kq)42IWCE14GJ{cvk4K=2a2Li-NRYa3iB=?2OuTfWgPY5PJ@4#YrG z?G44bNL8`EN8w5|)TB}@Lbn3%6pu1+B(s6d#a-;1MRTSn$4(HaWZ&vv8y-)GOL ztrE&uD->*7GmlEhxDN0L6Q|UI*%#5y_Fu$KUc&D)gg1A- zYY(DY;!L7D4TysEyXHf;y!-rxPk!|mqg+ep4_#Gc<>J*(?rsBdQHwftQO7)@xQpww z*vM%xUfwFRo62hoW_O{2jsR;w*5$W{Ix66bW_>O1`1GQx!RGiY_tT|kUlRy&fOW_X zFT0c+$$K8ZFG(|ClHucL%I^6;A|ZVRx~EO1;Ri4KlD7jHuc?6iIq>7F6%hmf8ak_L8g(~lVxtC)#J z@C$eVtB)nVtq_S(6;ZhA8vx!krfMa^5oGUoP@UV!&5&|G>~?vu?6Zu%gXKy+oxkn_ zMqfL3bXtt*WH%~KeqX#}&% zd&=jLh9C5o$RyiKf1oTo0izY=%zbV|`iWDl7_Un7&yDRef) zb{UE4(PK^{CIi=pOzRV8*)3y23I``*pMHKi?UEkFy`a2S5I-7bpL^}G+t6>ev^6(3 zUv=25QD#D)d$|YqztgL7mp=1RG8owlUXc z^2+|9#LD1$^WrK2-ZH~28|YgYXOVhev_O#CwrKvKl5FI`s7S9-w~CwUsk-!L1|mp?5QKy8b|BU|~lH2e6O zJHJ2gcRs9XHSm&I;B-P3&Uhoe3rfhDr$rBAB3!*ObJ)AA1?QK^UmsG^{k;}@x_crr165F=&LqaT^n6GK! zI(+YBT9*q2`&_a#IUU>B9wFpp*FsNcrB))Z)E^d4&6m9^UmGlDxqnIB{=HZO=TI+9 zXG@F;lWtP??UY`M-IKDHcUzq6ZDL{q2>Y%ObIJn(6KGa%s&xk)kcgDtlK7JwO?$d>Ca~68if|=b{0SkH? zVLjvrqrohUC>4MVfbA*;`%`YsoPmX@sj1Y|D}h853KcKzvbiNf0G%#AnXc1t@YdXg zO%3;V-sjUZB+u5%cQ3ZTTT~am7&oQHtymKZFLZ;YpHMUxJ$ge!%UYK|1|e{hW*8%^}Qi%G+K z`u;vVzebviaWL4FwVk=-n+bYUnA*l zAS1ucV8ImF9c>O3AWTe5oSpkdH-?iv6-1wX|Mp@()u_i>8*ce$XAFI$QzS{%=-2m8 z>x))9Klkrzr4hEC-_ekE!tXC+#NZ$8-e)*T2|%7*PaLoy+Tq^(Bibp)Jiv(qPs{Rq@cBdaKSPL=`Rcow zU9g%?w^|qQ{ke351SJ-CJFgED_5$vmy@2c4HF>>*w2>-)0AehAq4%mHJHFy8%P-Hb zr@Wb`Sf)Qqv+vWpNB1^BbtFC~KZs!~#|Hvvm=xsl?Q~^;DK3T+{;LaU3>h1KqeHU zh0bKlfH=OTYI0oG55ElqQiWgBv5a$s4Pw|w_A3YNGpyo_kWi;eDQ+aa zVcP%6O_j?vgN9Pw1))RFH`3Hr)mUElUBut9I0jizOrJ~_qZsA)hnEFnXw$DlvP4qf zBIhYKv8fnSI^>FS?`N~@pm1VfU?3IExq)){!f2IyRAi*ul05#C>L1D+4(rM8z2tSs|9d_@B#l_bh$RH}jir69m83D%#P@0}ukZaD#N?znfr}XD;nMJ}f zH$#sUvd=EhzS8B5n>STA5!Ek6HJ?Ux*KLTyUDB$x_?cru@_XH@{8DC?Gn#U(i!+C| zeyb?t4W;>gA7&`o5I&G}nQRQDVtw}Qi$C2a0Z=|jDTVd=*miu`JQyZ;-FTp zlv5=zMLmU0>~arPH*b8U(*9Os*vn6_l09>_&ik!mZYy6dtkk3Wagmv5OS(o(xOhG9 z(qf5qYH`6?#6x>ii$KK!sqMqD>MlML7s>>-gXR#5Z7<>MdNcCu69S_Cm_q%Bl=1Fz zr0~lPA#B)U%l=wgsK?g#F%XsR_|Zs`*1Pc|E?B_w@UG(*2OYr$SXpa?jfEJxuVB`c zT_)D#Y`+Xn9P3S-wYppD_-OcoUcM2vO{?0og7T1Fm+87o6PF}M>h2_Dj%ainFl)tn zNCU9YLzG;!(OSqtf0Jh)2d4a-j$sKz6TtJ(-E<%00y9HdV&zMsZDyWT#O1z=P}T1wY)OrI!<5! z5TZ{2srx3L8YfF?J0d=xb-YNk&VT+K(yV{*{jll=T(C^FE#UK>E_4yMxGDcfkhy;gF?N``zJO-pyu3=aHg^GoP{tT%XR{80agyH1)4}kN-mbXprEfLnT8N zV>-9~n#m{jc)!1l>PvWGRNd?L^(5_vjJiSV*pflN$hFLaBYKahwp*0e?6l_SbwWob zo>LY*^Lac#w#2V@Xjz~Ly#|yjQb`-v(b2I`_JZOaj<522(0MMoXG-ob?)aeT-UW?} zP8*EBKA35BcW<6Oxg`GbrZ#ar&xBt)T`)Cyz}pOcoa!^HMTv324>Y1!RML~na6EfN zroFwaw(QY0R8DGl?dW^o!RO7gmVxp2H@V21Uw9YABXv(VtoSFnq)AZMMnR=mes(9+ zZG9GO8~|0HJE2>;D7pLDn`#+N-iguuHnj>J;dDwLSqxuZ z-)_h7GmZ;aEf<}Qj6?tBb;M7T{KuZ+&9A1Xlp%so#OL` zASkpgwih;-I?qDE!|#ZaAKS_9f4l0c)iTSbhx3P z=2~aW;O8_&hEw)YqxivV1u8jMMeAbo34OJk^kL}|L|f#Y!CR$1&+MSY;}j5Mpoigj zcP@v{azJHy0;0nfJS zW@I$hDaTedli<#S)6~!~csAxkNeY0}D%6v(4wgG+>lDS?8OLA!@&tw>%r6tUp}HI~ zDp`7NX?s8@jbLUnuD?;;K?oI6w?+2( zf?_-V+V;;2zCx>R$a4=G4f?PlzEu&Wsv?ris_C+Ii|2{rWi}VYn-Ohn{LHLo26@X5 z?AWpDPN#SqarU<;G2wm(=|6&qX)YwnJkXp}_U(54Bp)JUyZ+_ZL{?38+J1i8?^m1M z?IvpEp~Nf}ZQWMllitOwFD|lTbqO-Z0gj4o9=-9I<~7^frSjE&tBjXT2$O-E9{Ss@ zWpp`@5Tmc?0>Nz$$;7Ln+lrDIS9~?&9le6QK4fCbElEo9**EvY z1AtDaB-KNpyIZLWQBIhvls>(FvS4;@f!Hi+dirL-jUNH9@A7nYRW8^)C&a|)=;?XP zrz88mCz^810!Q)s{#Td9FycFopV9G18;L~3W8Iglz)tYlQgOeUJ{{$~{d~LkD~vr7 zW>>IAA%qZ52R1sHkhHz{a(R09#2k3l{}PRP9iSinfD9%P_x{81&oqMp`-_rsln`Y= zeg_FbjyNpJ5xF$m|7UpNb-;-hVpz#pcnAOsPskas=B&?VynySePrPAyO(zhLD=WxW zCJM6HXSlOh>{JI3v-qhbf?IbvHULR@@72G*kb6Vy-v=?imF}gvkl#uKHBrlZB0Bc} zAAr~2L$W{I!QL&UqQE6{L!DrZ?fc>V6!0T>jaLpQ(z&#cf{U2mMP9k? zmLGjFL)rfgJz3CY1Z2e@@0GF;g5ea>Cj`LYB4yM>y4W>dk#b1KA zf(XoGMTP}`AM9Qv6Efg6awFz$v@_mG0_9@;T*D<1nUYF8>M^e_DHN{CX4Zclc&f47 zFGa6E2)Pd=ZioRtXz zeqU~(&WXWy9&gEbw@d?>%<>{?Wp%%&r@d^K;D|0V^I;AbfV|VD*d_mGvTA8XEnC&I zvSNDIN;Fxpw2`RtJy)!Ak3Yz;X=V~Ddm3;=m`nExk~tJWqx&S@qyK~A`2hqHQA+)HuRS)G|CbmZ|M{a_9{S{*AGtb&C@xQt%in%RA?b%PC;4O))zYE zMX=W}F_{JujVOK??i4}xXYXf(xWpZg4?{dbNafZ$>VKu@7vGmZ^yNWu*X48CBANJz zql5I&rYag54BW{edTTYVJce(uU%a0!(B)jvQH`L;qa|UzfrFn_6ekpxE{>&_8|kP0 z!ZLV?{vq5@$7cUzbsgbSh{P($ZqAFE`stQf<>Pm&cJAl?n63+IY*XhgFUE&bkc^;6 zbD~+0I{WnRl+Ledg69rss(E=+P>hpm%qD|+OW^Q*moxn8XcgC};tA$7lNwccZTW+P zV&UQm8T-2({vJwk|Cq>q=~B&8lPfPn2T!=Ow6`f-N8iKyEA6#cAalUeIVUUmPSJyI z>3q9CA`|LILg+hJwws=%+bNj`#E1mvV(;=vs0;Au7VI{^cvDR;NtqeKw{SrHK+$i3 z{BHkBw(ce?`P*l2Jl2a$$>FMKf_?GNDG-~0GkfBkXRgXRUT0tQAWy#EUnLsLaZWov zf*)%uDASVVd}78tyPbXf^Klnpi!LX;g2HLX0*9qV-<4G7{O@XTEsZ(i=tY?yP6&1U z4G}Qi!zI?k5!{-K8z70@)%CWHgu!3{S?a#SO!`CBZ?CdO&ji^=%yWIdF~aU9L@juD zk5jR{pJo<4y}uOkbU{5{!DWw95ppbiDqdm36wy18Jt^1u-NReDG-hSWIn+>Xedv|K z{*kB3_iq6&h`Qh5sjv;a>GEjRumKPXN)TpK@EFICIuamLsrf6vQ?fHaodY&j)y_*X z%rJ$YQ&2vnv?=T|{iKX&S5OOgugrtcd;07Q7MdSV2e%sd^}SPgv^KpTo9mpoqOo&= zvAPcBOFHeM*BxNopgy9q?Mi3GbEXQQ|1IKr&;<9ASk0rAgQ45A-qj|*EmnSm%Ce^| zk$`O+vTPwlM@J71YPWo!N~bf zMHsM2{1>K*RayO5+Va3veCXw$rTzj)>;NF~BR0SvhkG@A%eH;B$wr)%di!CfSCz6r zZvQnE1YQm?OLS=1D|5=xYZkCJ!xMnDSyILAInz+$&jk$mD)poKb12*YmxVB*|3vnH zNqzA7Cg%_TSby`M$sXY4|6wNOIYQSz6>lW5mrL#uAYl1UBIrg~DSSjKC9tCXZD=V} zMc1dIbEv-=-_vF4MX8dwX zJX|OtBl}UTefwT+3Z&hx=`=Gb_x61}4#xcd&SjfyGYL?m`>ANGM#|w}Nh>HQWNh9d z0c@C(nzf>49W-S14uiHEs|izfTdo3;c*(g8~0ar;8&jj*Uf1Hp?XG5U_s8wTeLxS++Es=UiVt7;9j|v~S;jm@) z6J4naK=-62nVg+N{hjkGzFS4VXahk#xtrwr% zhH{>eblvNEumu8|9|9FrQdpUq!Z?js=inVI6|r;cjD*r29TUv8#S<)ei6Kgag3`I9KxmWZ+h9?s;AT1~Zck<4ER_&;JaA~R!yFO)Kv+!l z%d2MSAWKGxtTl+SiIVP{$$qS2h@G-vM=;<87?A($x7VESZnAoSH`Ue}1vh<@z*W+SL@>;XBFSBIEU~ zBweOvqo5yA$*ghC2l=vJQ9&lV(A)6d*A%A#YmK}-gP3-uakpfF^K_-u8rE?k_rpAW z_y4r(-^BiYBE`D*e(22${7p*EPOLaSG0ty^+Z>EBC7sa&oD9ab_MLIXzm{~{oe<Q` zo*quMH(&of80cT4*do7vERwb7-dy_9RQ~bE7GxRr;ntzH;lu3n>cV}eUs~VI6))g? zT5Wp2EWY0Ry#i8Su!oy5aH4%$;B(&UYb(PfH<6e~F2&ZVtMV-4wK;`763pGGx|N+p zj(+P~U#kkWD!JI-RUeR0*G&-yOk(6Jh4HR(FRe1r(t5VuDYuN8VGSe#|G)@%5TI=I z*xjB1ysPKZkdR1qBw>SalLHzwrmA(b|D?pmiRb_VS~59Qi& z`0g|_E?236iqaL$b;8Oz3ER-yTpInDo>dkgpGes6qNOt$;Slfk@O9}q+7-FFgOfhr z9E24W;WZ-fIxE$-P0HK^-#eIDmuzD*g7!FvW`e!>rcfGr8-?zP5SAx&2Y5I=Nh|I4G>Q#${ECl>fP^bA)DHI#|6E;5-@m85PYxU#unQ`RX0j$}&(0KX~ z6VP@95wbz%=U&v0hwk8*H-M(*b>#o=mjWP1L)^4J=op(7NC4R!Sq1G0iUnr@x#BpZ zG~8)RVGaIJH+TT*Rt_c<J=DILuYzlaa|fA-7JJrugE?Af+u23+kaC-NlM$fzLTgv-HOYgflZH+Ux50sxr!5mYszyFS(MxiPAr? z(8(4Quu|3Tfmxv3lLaq{MaOf)9;XOlqkUV_^#Z!#Q2VHPEf*;cKgF6(MfijBo~)7- ztN#xiz)u@|zceKdpO3*zoTYExzbL9#Pg81Lc)>e0HZk6X-^3ZK>)c0MEYNwVMySf8 z*iLiMfr;K)bo8!MMoO}~l!%r7{7{}w)pxFO#i`6g`tsH(d@B#G>Y*xdm)2xs$4xrf zNz$q;bWPO*w=+ua$NyVaASWK@Hxr=#h~Zgrh@qC=)Y&$MVzsa6qs3J*Pk6<`#N{#< zvn@lOwLMs}@acLywoZ^bK9cD<@>N~K@)R|H-}S03d7X4j?AL`6%mLMg|5}&-G4s9I9a)0n&;-Ant<`4M6L% zO%dByIJX7mP7Tc=p55zv^tFMz6+H zk&gA9&h0WdqNh{4c&I8}AuqO9_|%cyi_4L0UxpVBM+TSo1o2}NO?J5)$Jz^@29ge3 zU3FIKNlp(w@jD8Wb<=JAV^v93B0$se@x?AAXEW?68deF4PY}U+ z6cQ_b>!+@X=q9$h7{9zTIs#YU#PgcS-Kpr{_J5nIbxrsE!f2(3;kE0U?Zt~_?@kM@ z3!#6dHE~W;nUtJOrTtUpUF2js?~Lvd=194z{qo&4 ziidJrEPZO{;U+!HH=((!^O*qBi0T)&t z_yR>OBtSfh|QEXKI}9STQc_K>U+fTicGy|D37&h zoPpSspmRsXtJ*b5mlBlQ+yF~}(I~*@GQu8OoC}$lH9)DqjQ%|ux1RvJ5%c-36 z6=KFm#2@Z&oL(rCXi&Dm84RtZc~qMOO}B$H0?38tWHWT?LU@^WQFfAyDofbCL^;a* zr$ha8kk>WzJWja`oYCu{r_>xutOr5yyWUBN&fcEu_h0N=zo5fszp-3%ITR)b`#=di zR~@i_3Pu0hFG%}#5TX_Vr(0@R!ixZ!yI`-}qO`=WmG}dXCj$ zTF*N`nL|+hJ9W<=Qc1KB&WDpBm)9xE_=N}mP#Ga2Y>q*)=zKc!%;64bd67eqKK~lC zfg&*!ER`9go9-G3TIs}Na=U`|J%_))#w46AwfhH=PqwW-g%-&hrI)Qu>urGIhiN<; z%0I%c;MEU4@8htMQ1?^J4DrH&u|Xn$fId_e1Bkv@jjMR-c_0o302YE3y9XIc(T|n{*bVwN6 zt^o;gx(c+%Haqe8A0xdEm!<#*Q=YYkdP96*puSsO!3&{PRD@%-oZUr!xbBDhcmS^^{Y zlzWrAx`5^zNE9f?lo(+ij~YBx&=+P9yXOwl`nVguyzTm&C5pA*dbl~n>Q1grf$&P< z;M3aA-7IbCd?4Ks4Rh$^6sO}mg?)pln920f4CO^i54Mb}Rr=4W$=$(w*bJ2S6dM)0 z^=2OC+F&-*fo9Ca@&vE4$j0U@>5{rKlv<~lhkMNX`>(`W;75<&aCuoqGE4RGw9ve}((J&Papa&_JANmm6CnWFc^K6He7>6M?h zq;h%^I22X6t3HcQ7wGjvB0_T%@Vf`v+BDPy7Bi ze(SgZp1Jz*9%zViT6q|m(2htHQ(fxGyC%u2 zg^1cD@C_cV#`Ggo%Jm+mkKCFd=<_|yRZyVfHg#O99ru~=Ro5&@$CRYnfjtgI}Li{Mn&Q+xXr|EmcDfLj3P-@rZ-oD0KTSq~x{ zzm8wyABxHCeQeei_t07F*#ogln$eob^?_Kc&|5Pfyf!mfG6F&7Z+c;h%C#ltV9ml@ z716@wf*4zAdx)4=rbdMb{D^bmknF{&Ob+!7X0uW29uYQr-Y^!Y40~)=PKw!SC=U;z ze@Z8*(9YWeKb}`;HE{k!$4H(Nx1_qC+`_b)1#AIjo4?x#vS7r;Lb3M2u}EIs#VRj; zudYnQL>O^!@UgRC5wTLI*UxlgI@(NWS=msypthrlFEmAP47EG{!5JokOhDeELxPiU)umt&X%}2;tv}8JWP=%-YFrKdD%RNn#*&qD~J+o8UoSDn; zk*`1Xg>JDqDT(!U`$5ZTWJ#N^{;EoA$I`LxOsg#J155AwSq8%k%&sHTJnaj#-WR$F zx=eJzfp9CNZ^RJgwLaU~7Aukn^)hHEwHb8Z@us33^&*q_tJdGj1M99iqIVU9xi#1d z3!0;oq)Dp%>cx*}1H=6UmrJtDpIA$B_boo@#SK31`eLUf2xZf4<>S{+IdiHb{Nq5a z1xrMrb<9Y5uGDR+cxzcamn|yYv3yh3BKe!%?u{*Ks8I!29sW9-H*R7bpDHj%Qx?dx zA{;m!0XDOP{j-0UTodWRbYymFT5rmn3(h>&c+^>It!|WpS~BO|iUU7Z&`G#cMs}hx zOWKKxd9Gr8WF(v6^-+E<59^w_R8=Koe5Hb@1lG}%yEs!QK$AKoM)+$HBAwO1a7#xq zfVW}BAwHrxkiKx-Sy9*GEpPAH#ARfW&)Xzv376XDWw>8PZ3CB7V!}*E9 z`rJG|Omm9@+;fZ#eS4)$5O1NTpI7-Sl*OvFvVTawcV6tqtl>C2$;Gzv?4ZA0kX3as z5xv=!Q0UT8msJgA>3n78v~(!Od3frV!lTg3!5=|7h49qTM-=EVQ!>A6klJ^-N3SB?&iTg1-_)xI-zX#=zzANA$5G5_+5WuWVY2zfq|MzHl zS;uM8Toy=|FHs`;8?#3MOiA1HSZ zWs7S6iIm!lz{hm#3mhofBx^Y|1!&~19)UbRse0zPw?is* zrJTUJ#-R;>m%M_q4MiCB2Qd(!^@{E{jqiaQ2S@bMSC=0o;LwLBU>vx3B~d?=K%k?3 z{KT3at7;xkIav3o006g;J{Ipqpp`nGLMhLht$@Sdri`++%8$EO@b1Z!DI?5DF+gtN zEwSn$6r7`srC11-audmbk|s-1lxNtRLieY3LB0o^#Q@)J0Tjt7IH#iSzJ~;Vdqdja zsj~Nw+CPlE_pb-R+=y9v?}nWE)6%KM0$Odlzv#6knSCnJR%dbH<8z)~qF*85=M2gLp<#P0V)%J1Ffz4t)c zjmN_!4oZUS8Bs#t3)gUUtvL6V_CtfGtk3Ipw`fwkL;z&g={uo#7b(V6KI2&1mzr}t zvlnP~9;Rb&brT**VxQQDFJBvSvc}GvqCket)=Cwr4~vrE=~ei29%EXb1Xf-ry#u*E zh&hT+UkseF-%l%HH~!^_yXVGyFW@2)Z#vdj;b8^_Qt!ay|DpEW~Z+SLa}jLEV1Vm6+(E-lM5?HFF!DOy!!y^5o2{zNB3)y;P&*v<-y~ zyQ!TGwP)2Bkyxt+f@ZSVb>m*3U6w93SIFjq{c>n;q8!S8r)sWW97_agMsdW>R*rrJ zD`E>I8)<=zl|{^Hb`F_Xy7}nbsWt6A+UW(Fmsa(_i>&Wi{LMT?MDUR)eQuDSU&ivq@T`vdDSv1>poydbxh}aJO&E5**mn7PCHxnEHfaf{S*b#?70&; zJ+^Hq7@?fssrmSRhryRu5TtU0AdQi`A4=kEuUKiZM$mF7Qj47ZpK+OojcI6{?I|25 z9V??HeCSJU;idRMXS$GPr3korD7}6aa{iiGos$C@92f%vK`F1@%3#}(0{j*gEo&ut zmk4v4yAn~Zf-asx%u&N$B0nH*P9zWb%(hjx5>xxPI8?ntI5wc;#+O;bT? z!X^bea9eh9t^MCQAX zUOAVrj4F&6>Se0S^pR5pn?JM~E&nx-Z1%dy2}e}B{FF?9wmk(q+A)89wUbmB@lhY% zmZU29iDq8pP0&$3{@K0{9f5<_Ry%9}R!cff*YsbjrQAA54`&v{z1|-cg7vQScs!o- zw-1;}Lum#LrB>RwQ@2NmjO3Y2R2R^VF%5dKM9pm7ZyQl?Zy%LpEF_c*M4R0fEO!Q5 z=p~02AZct=de?w6v3Y8y(85Vr>+7qfn3A6J%s51EIJd1%fWt$9_|(80*YqMPC{OuI zy^&9844j>HdrSSn<^KY*2}8Nh9A&&}E-Ra;hg&e@SdC@l(N=WNsz0b8O;L^62V^-r zQocU&JKOmP1VUyxaTMFmz{6eP=d6?%P0`jl|IMjA#ap-rD3{CkEqhaW-$Y^nfu?$% zkS0DuQ9_oE{guFq4#a8pcW>+V&Q`UqT5M4gkAU_t3;Xd~AZ9s=g4ljUO^WE5nEH=s zy?68qLBVK0qBBJ&jfVZ_k0<%m^Jf7l(74yGBp+cF9AM{lKV#Sz_E6+rK<}5&oK$0C zw#0t@2PyYE9spkgFpgFW3r{Eyq@3VBM!^U|ou_U_)E@!t&e@Zxadr)Znw zcg>|xQ2O#HigrthESQL4J3693k)7ftNJA+#AB@+%@S3OKw3Neb zEEv#Jsdi_dGc>0Q9s!-}uTpmG7PX?nsc7LvzCJIfo%tsvgfWHtz^Xs$Xpz!e{TLIfoZP znsZ%gWh{M&wyiZ?@CC3q{o#*tj$Df8f2&inyQryy?YFDFPXo{aDgK_d7PNw&RpF08 zcdI0CeOJ>Dl6B8ey760Cee;jV`B#7a83AB(`9EwH2wGE%_qfzdqGsfi*q#hqTU!I$ zCGY{cfts3^O6#AByMyq6q;^AsI;@kV6Ec`L%JWA-{B6#CB9IoejLPY9?SQ zdc?V3f7m25ci06S@jF~`{DXaFN|o`gTuT4J1%XgE<;0%NcvjX^(!T**YUBCZb&%RCLcp9sNms$M{yt_2cxQ0VYLfw(;pwr_^79#k6 z_wj%-XM!5Vu;R}>;Twva3_be4nD}4-`gTR}c+v6Nm}uD-^C_q!nVC`M7Jc#l1QUf< zKJCu#I!FNtRq<5&OvzTbcct0HE#38v>0SmF7mE}N$-nz-fWEk=u!N>wq3FBIJi=8p z2jPs?^0`?CbWgV5*(rLR$i)_^2}+6|_I-&>c%Hh5}N*s*h4J-d&dsiOUbe80GJX^&J#3L$Rh!Hsy5CtJBAQ~ z|C#@i{PM1P^{RgLs_I*JE~ix=aDm_@|E~S285>Vu^72ni_c4l?tEZ)=MX#ogT+{yb z@*CIrXKtBjKef!T8`MOUg5)|tkYW4Ys&F*({0`TMP2cPTHxE& za@idvWtG_HIa^oB*4rty<~C+spV_MK-)P*kyxIL9y`G^Z>KY{}(yZK0T>fob2T^GD zn#V)A%0XQv$ChSAO`V#fUFz~KzwTz^uD7nRmQ401bg_7_WPShN zEg9{jx=EGU;Z@$sIFsF(H5SZ0uUaq(S2?S%%|HHDU866AWpAah&;7PzUpIY5UxrYg zd9KCn-AMI9fg`c#+^XB|X9C~=gP!mE{7SCOd+26l+snLWg=&hyh64d}8!%mYcOe0K zXm#*^t>w>Um_-kS@4q;{=)PiC_U$_VN9h?covgADNh$w*F@!(1*$0}m%!IQA^Zrfh zrk_TN9f)X-f;dKz6dHm%h17!}c;nZ97nT3$Pf2+s6y`pi=Kj}xTM1&mrB@XI+RV2J z|E3uNw)R)^uA=n$BrJ%EW>Ce4!@6G(G<4t&;24q~zx(M{JgCqOVMYPp^jIq3<$>1w zyRS{9UNa6On4Q0X3HQ+;v^n}y@IMDP45ax#0xte{Pk-5y|1}Eqzlr&;C+3ens>LI; z;`~~+!-|#ZyLtP3OS0SQ6!?+X=NB${rWo<{qFKrYe(nu+yhU$tS(S)*L$E(^?AImB zt0FZ`>^xM0sE%H^cFNC(3cj2t_lmbyQSU2Rttz}tA5_qLEozApP&e{_jPMsT*>@mhIoIdY3h1r?zKl()Hc_#Ejdn`}ef^xyPTdDM+qWq&*4X zecYj`lI5E4II-!C7GRAxMD95GXrp{$$#wgIOqn-d;*^=*Ij5LYhk}pOnl}X!7b&0W zHxe}RILtt0lh>wm2KJR*3CbqrWLmZbjNIQ}`(7CRAw0o)3rL!dMh(k~yX+F$y_>`W zldve_@+~%(96lDs^)d1tY6Ctd^~@sZ*0qe3WLoyDA2ZNWY}>t^*n?#jR?5CcYkH#) zEuF-H=21){&FHYRT8&eZ3>w3*UE)h|{g=lODcw8`w2U#<-6Kk*XjglBz@ z?zVM`=mj#`us3gCnZz!syNLbiuz7f2CQ%1d)4Kg&Y4bko6Z5QhKkTI^7{n16`D31@ zN;x4bIGVh(otI>zE}!%ioX308o*H-owKc37^gU0^wzy4r&Ivkrw4UT^u>)gumqygL z@lPAHxx9|tk*`!$dB*#eYfRmyM*D6;SVy8Jk8RD*(v%9&U6_tR-c;* zsGYuV4Bd_^jp;1&5u{Q1fDj2Vl zeZF~o!9=odQA!xwS+7cz8ih{JY`rG4dTEvK{fvoSlLJEQ{U;8wHs zBL@QJv6fsQ!s2^hd*QwkO(x&XXaGDsr4Q;e#X@ODtgK?C-C505O3=TG`e9hvYs5O7A#0yZ;7l`y#HvX77euX zs^~G@bV?5#n+qS0z7Kp93AP&8!Txd$SCO?{egn(XA-fb29c!o9t@VZ4h=iUfRS-_AzJ*( zaIlCGtf0fSZ9K|$DGZFzV}K_85IxrZ9;sZPslE@j-2B~Z1P-O;&}n6~1umlrods3K zcNiaNjIjT%X?*`LLgRr6-I$(`XT}@;F*`+QyY<2*Xlsow8&?0WtqBZ#dljOV2mviR zZUeFk^kHdR%z>PaWnF&p9 z;o)W@a~^7@lZ;~Bdbe<#7fu}6r=J;)O`{YeioK{(pG2WV@Vu;}(P->L9hwD#VZ zKf!!m)zKc!MBS}1fSJ!gL)`f9)AT9Z{ZZ@vh=-tN z=byxC`s1z_Ao_cuSsJ4??nP_dY7o43+9N1+7iVv9))a&cL56odE{iuvAX> zR6TkZU#nh6$OG95f1S_&b~S>)Jb`o1o2G&NO2|a}*<)&zM8Kv~k{(H+mVV)15OgJ6 zr66AWg^)aOY{6h?##yy#mOe<$>$J@L#@=3~=eV3b>6GVI(~l~&Gj4)VC;xCr9(})9 z@s5ypr*e)rn!wfjpC*nyPFc1>%LF=|Z62!G%cvs`J_Bu0{_^MzV8^}Og)lYaMe+eO zYwRzC=~J>Ld}8jJHz0Bxj*`nn6E@^hUg=fmgaV@m=9|@73y+n*@>VWj_W`TV)o6S$ z+Acrn$iCJxXZz4Bt!0D3dqHy@z`Aa|tDGKr$XLd$6c+1n6d(UXMnq=HGHC-FI-HDE z(Kk<33^}sTywq=O<)EI@Qtx;a-PT&O)BG6^piZF+m;diBpu&hIt#c|QxTk&0CM2V0 zr$fTzRkNVJK1~|?{54Ts4NIsX=U>Nt5UI$JTq-CqEpvV0y>`%R!3jfrd{S45KgHdh zt0y={0k<&^ z!AnC%c0IU}0Uh8N58~28H~`xXl`!~Fpkwg7=nTi_@*RP447X!yj^N2k-$8b4677tf z!6~c8n^T3BWlqe*kWVWOlHYQgs&00)*mC8l)CcYRT>_qFQ)OJ-S9-A|f|zbg<@}%Q z;?{M%!cA6D+&y@HyYi!{gXN^3YNb?I#Nlj3?mca?#M@{^Peg7Wp~A-enWAaf758hU zPxLGcX42Y4EAiz|wz&j+swD?g0J+IbcQUn9OMF5?ZZg&IX+b+qkc;Dlmq`R=BoU)25%KDHd7qd=FQfHb5Z(XM zWTaBalB@+VQ*G8{ucqx=>6#1ixX<07?!qM zXn822Y8JG@$%e%QwB?t^c|Z=DNIH9QBM3IK_HF$EDfH-r`@B%y3zJ?3maS;lhB#eUoxCf3$b!LGTm?x8uO@s z*rozN#U%Y?apiJh4N071P)7YC;QE!F6X1N<(HP3diaCNKRScvxBFUbAxIgZuf+#n&#}OrLi$$8yDK zB_8AuoGv79xy&hmpY6g1l3Zt=#p%i54((7SB@WCNif_T;M7RoE1vAkqBE=oH6nV5h!Z;`JLsFHQYOryI_2uvbZspfod@6zg4zQ9sw%TqD2l0>F7W@Y+%o5GK(R1SKF^H_=G>Va3GuO#bQrz-v^(a z1cZ4vPx1pYYkT$?vJhuyF5skWg^fGe&xlZ37r489tjYk4ojAj^uZm%I8iKp&Ew=4p zM^4kbf?YknzJDu|;hLA%7yR_%V_VlI`<0f*7I|<&S2|{dpVTpZs?^fYz>HQ#|AJZZ z{^WffeEj+nsDA+dVR0Um1M2X5t^<`m3SJ|1)MxiPIdv=I)9Q*9XV&fEoKebblQx+`+3)dnPO3A2w*nQ=bR;5 zOT*TvhSU=hcFCF<3>UTML`@W_Gh}>QhAHvKbA4Pj_#N0o$2E`N&2d~dUTH7Gnp?rl z>eSah=hRu9mez^2Ne$S@44#U&&>?2`RKCfnS=2VuIyrUd$I^j3Z*iPtjMRa7cXr## zLUIO5onnvr@8FF1EDMPpmJH!U3W5obp(c4qViynB7j3b_*cq?8KFDe-6Z2GpKSxZM zREgG0Ak~WIjwSUn?1oX`@97PEGk!5v+j&-Jpd=4erV%MKJlHOxz|3 z0C8stK?4*Dz}y83E`Joq8hwq^b`wHTvnW*0%RT7l>7zGKwr`?q5EWy?iH6K1&*g+a zhVc-N8&;6c18r*+4PdhXPY_x-%1{hl;lxtv^jlmI-F2NOgq+OKXmx2*B%Tn)JQ$Ly z+vICQo0$N34v6$^VAC$bA5r^=269XW?F=uGmn@o$_aH92y{#hvEQRv>|B5cy0Jm?^ zZoH#-ww-a})#7q)9D2cf1C?WR8XL*qUK{UZFSO;s?aLq5xU(2bkrPS68p>p54%cmf Qx_&c`7?|qQ4mw=?509s8+5i9m literal 0 HcmV?d00001 diff --git a/docs/pipeline_config_3.png b/docs/pipeline_config_3.png new file mode 100644 index 0000000000000000000000000000000000000000..2a3057e847e5d2bb974742a9280375e06db919a1 GIT binary patch literal 22416 zcmeFZc{r5s-#0$hr=lVel`JVy)@03^C2MvDBV@}MTVkxCPmPFB*^2D@EY=xhEo%|k zW-uCxk21!Ptc~qCr_X(VpYMHtkKc1Y$MgI5+i^G=mus%;yw2;rov+vHeLgWW(PKNp zdjtZ3uo>vj2zxv8XJ5BpsT%}hbJ0LY(<;P~PT5Jen$6nV`*m2owL8Z$(&87iNQ__) zExOdZQ=XDaX|wgszx+O}T#?%a9XJY;;K5O><>DpIzxadeUAV_zCX43{53vnC(PhbR zi@W~p#tl7=6C4j&4s3=ZTPs?=Wzd=$sedlMTFm}l&>-^t)_3L6H~rMb4UFlhKN?9m z`weagjpnd=Kqw^n8N})gh z$_kzgeRuHGTuGWX?M;TPN8)2x%l%N-n1c`C8;cZl*MS2EoGAfc-u`(bo`+v0NCKr=K`UN?(!z_+LUw2D1O#hPVr-rB8B6nvo)!coT9 zThZ8Cd30ReR#JFR@|C)}MhDOngB5>M##=Sx3)~2dlV-Ah@Bq%t9 z(+QahK5ZS&|x(o`o;X|VNVyW`R`RnzFUjQ9)fmiU+d1gbnA9?kQSVS&3e-F$9Z_^C-tqH8`@J_lN%{1wbtI3 zcAerN2(MIBWn1vBi~rMA=g{`YY|a4`R_D&qlfh4>mxP9lms++Ki_#qt37r<@yX4-m zo<9#9K6xB*|9e(&^(#HZXeiMnHgGKD#E9hN?8$L6$d~yaey>b&)N$Ts5lx%Z@uL+^ ze*6kPq$5qzKCs~S1|Pxb&0ImV`j>cKpKrmR{kxm;pyjuK zbAMPdVCnp8VZ5gOF88`TFXUurNTt<#sGjZ2k`VTDFY|Tk*UzId>x&eOHRdH_Okhv< zy?ST2!_c%>e+CB?{&Rcf6Q*?sk!Rb34A09vM!W3S@~gP>Np1g28SN~LLtKDSY{b_3 z&X>a)`gQxS`}hhxb)I3t!B-7ah0>C31_72%XD|k>fHpkR4p)x#JSG&Dh;q z^7{3eb9$uAvgS_T(7kPkhPiq#64qSN{@{aBUCyUa=DpoOECTKm9p?xBVP7)iBhuA> zHq|C_Z)>XcuNOxDF@V^3sfM}gUa3alyGso;sVy%XV#Jo75%8$5h5a^85o;jqy6zp0 z-rT^-T#@_Iy4~9fmqx}H?S1Cd!T?*vAE4GNm%?AnU)O@X=Y#h}A=?q-zXG_lpg33& zs&X&0ZLmPk<5Mm=50gbrrNoD|;93#=Ux%7ECd0#YS7otDwLDSlN&G6cx8FSk!B(@^uW7k!XJg85 z?^W3#@#(8MMunJ4jOh6Y&LjafGKI49vsZc|%Jg7Xp)+e0n7*8y1&=TM@zvHio9DBC(;N>k_xyP!HB$xHb=Z{CK15qDz^ViWc0>(FzK z_M!8v&ztY*`iqqFL~#yYO1R0={BVOlaf&3~|N7d>pq?Og_?hFP#@??_Q(-j8h=KQx zjyVzQdyL0h2w;`~Gi(ry^`Li}LUV z>SXxvDG-mgruHJimB~h#Bbt^Ljm@*Wt0fv`<|>i9zrJ_+Dl7~Y8BK+L=a-)sp%O+N zhaY4-EtU&kUjSKo48G`z?n(IuyTc`L`gF)jpCX~)&gO`v>DtDImy@^wh+0^-#_$c_ zhn>vZ)~3uCA?lX7lO|*RIP6dGF82gRNSYRHT8ZXOVMJzS!Xxh{1Om zt5jaSUgZkr7(a;S+}xWWE6Y>Psc-*iS8hmLi_zGA7lb!LtLTR-AdfI3DE;2*oOxER zgB)`B^YqvAncqQOG>Ww=$gXXaiq87hlPXR>Q?Y~MNh(L9gM1|UUFXo*fqjOj?6k|G z7=eYXRl!S>Ot)>_o|T@_koQZUi`xAjq$GEHLGZnw6C2zxG|5ZnD;45uW;;++MW@#RvOyN#ju4;MQmLD zF=FR;txu<<_!a!V_3Pv6o7az$(K7N}^!)Vd#}`+rT94TDL4^v#5Pp$6BWE4C2DU8y%V7mI1Jp4h z$$rD8m64hiU;b3Y^;2SWOc;z?0bz4(z#G|H`u)ZuTO6K0CuEw{$0B0z6}dx)TXSW^ zTmE}VP9!XSq^oXULZ79L;}*Ka@QKS?-I}ubHne?$=!jZa0%{;RpKI`-je_dW_ZP%G z8g6xll{f?~elBB$|D2U+8Csp5y}I56-=&RK)Y%m4R!;l!PoonmJj^P1*f8mT>XIHJ zWyi(!4v@#Gxypo5rznS0CF{uohs0_kkY;pD{K{NG()_{Vn%ZjzxBJv+VOf1yDx>O!bE&wbNSCEE&4yN& zS1pDQTZD&Fi&mJSN5zsKA3u4T(ZNV>-M&Gwim$1_tulr;!{lSuj)db%c5pXo1UN>w zqQAcJ@lbfKQc8!fg7?#X3B@uq)VT!oi3PTW>~Ce3B}1&3XVfenjgZKnA4D02#fmSQ z-vyqdmj;1K?T9cfEusPIUo4h^Zew`Tm5(;taAOp5BguaRzQcX?a!Z9_0?BAJE@1Gr zd_eOnBnyFNS>qxB?K(e;DL2KkS7GNk#Axep>>pv&vg_}ZCsi%qzmQqd`QTge(~)g@ z;KHTebm=;AKJJ`>=|L}H9m7w>(VEaDthuy4MV2ZB<*FIMyHD6GgqjPk;nT>h&PYqT zf?%Vo5p`gB#M|1#!xYtQED-W#A8`?3+JMUlbgD`^XShGJM(sq?iCkXS@X zS@RI9{oYgMLcc5K@*`(0PMPJ*SXI}Eo6X#$+nynGv-`gbN(nh(b>bwqvb^Nnf!zI$ zqndi7D&v!1onx*V8wlQ$caB&jO*p!C%|CM-sz)Dm&*K~<$Cuo$rh0zdyrxDe>KBzO zkK@53=u|Xf^;=)&8c(HkH5P=V4$Z(Gu@oK%x>7l7^{U}=q1?l+LHIaAxy23hoIT4i zK`Ggr*x+9M#(k^nLU8~|ikj%3tV}CmJL?(T*@QIvw0g4NBO9*CX2Mcnk7D{xPQlvM zAtOCOt2N#@y!>Q7!dgTA@ERVMZz>)pNLs0VGBnyi7%SBmlcBE|ySNKTJp1l-5Zek?W5sVA0Jr|z6^R=D^irvNEoEp-u} z;Jte0O24esSR$wNLyB^&uI1IsMS(01Y!S7_N$v|vq!GIl`4Pk6u?V7&&nP_1=VVM@ zafuA_VBis0q6f~#M9mc5AO;&i6S}Q0_SJk+$IRAEh8YFhFgCq+kR6+Wg|71lDk6?R zUAJ%Ycb@4=HvFDbUOZz|%S*jPCF*;;C`Y`D7{XnpuVn&uiyz!l*@?ZnaW$1 z#AYM>D{LEJOgGJV!Lj~xD65`3gV#^Axj#^lx79Ea+wPjAR(wLs@|B~9U4kr0qxlrn znP_CWgrJ=8N>(m)a37Vtg#PARr)277<3Z2oCrB(Cr}C?%nv4Y{IsMcL!xYME+2bBP z{QBm5c1^WQrm#v{82J_AF8_@uoUI?9Osm$g!tWPl#6?G#j9L*sPfU1Cg_Qn5GZ^XB zK3ZhXInN=T6%kJX$J(_kAj!1j`31fCP_1F>K`qszN4b3n z(gWV#sO~p&5(95GLy__$vE5H1=gW4$o`e1Vwn6`aE5)I zFR@xZjhr(#hbuy7wKOdppcW*lqwC7kKrM-+=kE&!xp2KCJIwF^x}%9Pk6BkH!I^17 zndaaT!*g~ZvZyKGyGiN+I_BrbY`PQLp<)T30ebEInTnqzFV%)u*(;kOEtxZF6vGD_ z{5*_T_ir;)*$U4_(ny||Z)FF$8CYF-FoDKa77Z56!U7a470c#i`4p*i_3!Aa)>fxc zW5GJ2M!#hQv-mD)Eh8y07Z3-#x&@=KHH~0pwD2@Aka&X4SMwdj>sqA;^|PM4K$ zePK(~=u2TU)b@w-&&k#it9M8u&AY40;on|)td@yIgqA&udv49fE!`{!U=%MEbQ~2~ z#=q9am=WA74Y!C$v}7;Kbo5ylB9A>lL&I*&-+>;sQ0#o95W%39DAd=s?5s}^kHk$# zW|c5XYk~$7YvY^CyyZ6Jh}p)rmKF&lYU#x5YXK+20X}1wFjnX1#OJX#Ph^aE(~G-a z#VLE8l1BtbJRm0LdkAHf1dZBTsi1G@4ZfBX=KBXX}xQZ1m+F zh!jofAD3z=FDXYe<_LK_;Xd_C1A$Zj5haa(rFs5UR53d6hg^obB`ukNcC_{5vLgn$ zFCRP2xnHS~S#YhaZU0&k3}2{PC@)`o&Y|QX{KwrO8VOe;T}EEsW4)>MgSugrMx)=SWw#)#mycO7@QE5+(Ku(tz3!_F> z_U)-=t#MFOz=G{`xrm6)cK!#$(BR;WgQsrgx)b2Z5ohEhw?pV~idBcGaYkXzx=(7_ z3_Ra=*A2R>{)A9k!mu|A${n*i>cVyCflTRT*1TB&Cv^=~oX0#o zYlLI0uC9(%TZdv6dJZ9LtTA8221tF=n*4aIF~BTgmCm@NE~`9K%o~_oj#A8mfEHaL zHs#SdKK-NP_lIQm-M;Bb@$Jt`7>*G_E&q6Vthuoe&!xc}9 z@GG+^_$M8OxOV7o3!KML^7~~>TvBa{OvEE^aZAF9>PLx@^@pW%Yg?&Oo zkf-i_hAfgD(xGuhs*~b{;qC{O%MJO&toanm5vL?+B|a|zM8=q@nl8pBfdy<=^>Y)J zosnvU{~H?zR_Df(;z&@HR>w6GGVzR(03*e+i#us$WdD7)pTeP9rPoW=l%*y_#HXMS zhd=%F)f&6kL?3WmRwPm@{jKr24p>TIf!$&89fWy!LqVVM$k+XmmTb3=d4cD}eIplR zQ|!&5f@?N74Vs$HEyD9obdZAgE#l2ZkaWg;@n{@*bpN@2Xv_fLv{F?j{5rDMb>G|5 zBb6vPe}Wh-jc2@>1@e?XD~+S!yM9#nUG{8 zK9cL0SO4^kPcp~O&JL&sXiOxvolUW6Yq2qpQ2Zlh|Mf?x>8p3}dRlzy_cui~ei@jW zege=JshkO)jxJ8rNqb|YRExOt*7%xKf^VHJ@cNlC0PV~8fDGi#9{PG#)y?`ss6;Ls zDmd~6oL`oFK`3ze3cvvX!iX5(9UcHqgg2H)mEh0g+3m}|C1rXAON!-sl;wPX%|Q$n z7J6XL)uZa}U@yFTdt;&|ujgpAOhvy*uZrd%#;T{6+}F!E z7F@eE`c>+K(gUpf0$-S^i)qnb4hpc!T)C2a?4_LN5P%_803I{t-_75UY}Ocwu#-fL zLf6bqCvLP^!cM?qa8?b?XgjfD?njw1JYlV$o~>I1kZ~ar81k%VN=Lm#Z~YrDZyT?` z<2F9QABs=F(1C}#@_#@}S#Ykp;lil7-7g>;?=A+nZY|V&ns0f>ixkbWuU_IdFm1&SgCm|cEp&@3@ZGCU!Lv#L#Hh!9CM*+PdMADM9(%f+Py@8P(ul-iEe z`@E?XBRZkj94C84H+9nO$$;`5ys_5%caIMHHBgH|RS~^b(L%2Gp8*ky9yvWXSOA*@ zVE2FC$DJTMGBRWcQ?WKmMkCzoU^z`{r$5WskcGxMRiIO@?=>GU3IzWwF)xz60!^J- zd!*&%HBsj`m8Tuk7N=)V`{gR=xr7=pa0D;jYqVVL?w9MPrb&xJOz5Rnex^*4`01XV zq_i70g`J0ph(M=M=7}M2qQHXf*aa0AI_gJ~4fD#;!In4RKF%KfukET_I@uIIS4Qpa z4#=edbU$K5R?d2Kl2wRV6^%QFe2oSs=>=eSxa3h&&l_+x>7xBGgV^cAKVH1chv7_9 z*MHwZ3O7DKpmu0N+>5|abCo#4GiPuz^oJ9sg=t0Q0#0 zRP0oeW*%YojgZXekY@@;v1k37{J(vC1t9gljyQlnE|9${D9z!)b15v$q4Kxfdn513 zDs3~SnfHQS5*?W%q}^3JrdaWT%&~XJV4CkIBa6@b^XE6qiWiE$z4O%$5;G~#^|&o@ z#P3<$0GYk|nNrN}`xqpP>;MFb=z~h@``R8e+D)~ zz^pPu#rt1-r%+nrkP*rMIApB#`VrIEp^#GhOM={a$lsv{{r43#0ibD}ci##{l5stD z@ej!B60=)f$3W_y=Z@=r$^2{cKS!^w(u*BW&&baIN|)`|3o+c6s|o&kR>5x)tHlqU zC%r_C)%s*6;vVdW;LoMPx#!KQY2Y9P5-9LLR0C(DQNZF(O@_=QR1Tf%PUNR8E>5;a zMcGz-dPA3&;Jpo7RUB0J#y}O%1FdlMi6NQNxzpoU>b9;m zvQV3|)Q*n3;@TS9VAN(?#5liYLJmidg zSHta#mmTibp=HwYn0P6yGR#A=4DX;pNKd%a3(fwltwVK$SsxyJs7rd&(a+nVYqlv7jOSl~`nb}m*u;HM9Fc!e{1P-1 z;hm6B3}ZE#0HkHrz=*m|3Z45fHb{r{RfQNFK_i%)2nikWKNYr`r^H? z0YvY-=Zfjs_>pqlyenq;h#S2uM7}L7nq74KM(hEy*n_m8_oypjb>hNS`>{M@>8v7Y zgSMC39$&?w7f;fnp%l)vT91R`lDf{TS2G{1J@=Nzk@>1G6wh>Q$8$qR@;xzocA^!{ zF8&|KUbh?CkzDi!-A3RB=l;I8|L*uPSwx)|CrP{{mj`f^+Fs|4Sok}ovfy@fs&x-^ zA3mi^6jiEQnrIciV()XW4sG6f5tBDRlU9jW6Sw7UB@0=MJHpJO=&6aXO)m3af!bE_ z6Zf#mbw3fkXA+8W&P5VlRTy+Mp5kC~9Oh%EM%3_Uigc#OjB4N)HqD z)O;!s7ib=ga2A?3oktQ;rWQ)KmJ}KpUTpS$)7q-xQ$-!GPIL(LeL@SbX1-&ER~Y|y~HCdmT>2;N_r^FOTH23p4k@4=0C>pR7w8?Es~ z*pOBMvEJt7M9ssZw6SqDR8{lB@q0RII5qK%<&lxn%j4!F*95dA(>P$z0%Yw64{cBB zgyQuO=@%C9Jw@vYH)Lz|!y=}wD6^}%#I)gGUUoBH*8I~?3bBtmJaitxFRj(FUq+e% z3Km5)eqQFD6|LK!cxhXbYoI6Nrz97`l4A8lf@j*iBtsy6^Zh`P*=t+)KDfMa$EDN1 zv~HEOw6vVqRjMdVsobC2`HbVVr=i|foWb-c6awS;AL_k-6n_8u0jUN6`hO?`kdn7H z18x6+oC8tuZ&nD%EB>E9=u9X@Fji}?N}@y{@Z!NezmIf4nr?3{CyN*>hy1t=M}&gv ztm(_cqs1m!Vc__{$bI`-?_WW$c(k}6a+a2yE??WC#d1N(m?!25M<=50@~eg6OG>`i zPWZ;ByYKDpB=_QPOF{;$686%lD*&}vPKv_r7X!Q>NF!EbW(QfnBD#C`=eEnK3r*f% z>0|0B07$%pV&!VtXkLl{oVBwyw=gOmNCM`0Uvo6L4n(wr5_NOPNZM1=ef8J(oZ}N> zfXP~FTIpvpfti4D0C4{ngI3XPQ6EHy(E(9i$EOx1w_1MRqiFEZp+f+@4h2lQ!~Nw> z#2|~TN55*|+%shSMI>Mx0jz(UPb26V;8awBS_9CBgrHGjWJSS;vEl=V*|&eDx8k1@ zfwo~&n{o83&qOUHjE3ViYiF`3WVEk5r`A>rfM`J2CrQo!+F6QUpuE~9+B6hxzB?Y78&(KY^`GwO?Fszk z%$nDXUjrn+ITgPv_D-yWSFd2anlTmdNv-Els%k<&H-E70t6CICDj877+h|OAb(P=W~5QaSn|E%#-=|(%yU{8Nhb|iGpXg zoz_TbivLU(1*BYFjUY2&u*R_E4+{?`H1Yye$=7`-)yDD7{a^P82OfCd)tXDxl%MTIS-hTE8h@Nx`V?szi)-=_M~BqE7dvf=>oTvXhqED4tae zTL@gJPI;`q$SNT;uMxM{;GG@hNH$7e29u0$8nH{?km}RCFm@H9e}St_eFmd7o^z#) ztPU1@XibJ^>gZMN97LVZ1az`MXU)M{Jx!}|B{q8KylCPx3!moMG78LAx6anin>1@%A1^l*1cqU2r1~5nXxuTWYqU{Xy{{HL`-r|z#C%x^z+JDJCImV9 zyHMQ%jRW0J8Xacm8$i%erF#I}tzEH*0X04PQG=)5#-%h8(<%AQYb1Lzng|?N zzs^@MV)Z%l0A%<*>ZL|hOONq$^{2G{>-=%nrqx}W)OsNN5Xvi!ZT&FzCg3}W6LJ~8 zI-m}(30a<|RkS*);S8bTWu$j%_$qW*C%4U9{mNr9D9d31?d*L9EGl;Kx;Ur`{{-!G zBIib&({$M<088I#9W*y-S|@vv*darer~!wEi`-T%6-_^Tp0IB5W7YGsm61Jhx1nd~ z>K7!V#R7Plcp`wQ;%6mx*$YZax?BA3wYrO?S!QkFZtR8Vk*K;h~1bwV*-dd2*jP;?(#>*61#^w$B?m3K|nvkNU~57`UrrWhQ>x> z_73-%2QuZ(tj?_(`mHWk!4!_~0ri#HMaD+dUa<9f%09TAJu={StF*CEUUWj-PoFx< z9Tl_;v~Kfz4dbm>)N4yqt&RtSFV{CJ#HtPBcXuPV9PlEopf24!RSsUiZOxqo9`CDU z{|_%N)*?Z)W(tg8pHUo*C#g%%0OE;p7=dqC_`7t{IHG1;O1Mm&s^gr#o1-sw0N(=o zDp!>>w_hM8y+oPq5Lz`eG@>Hm=t_o{wE;$q1SQQlB{GX?21^b!C{=QcyY7-X_HhrB z2H@GE-LzJw-|LefEiKAM4Ir=!yk7wYowtZ=*`h4L)}A`AFrz1Uu^>ziqkDowUsf<< zXEg?w?iBeZXuChSB>*twUIdudv17+RW-6NUSGQ_g7wpKV7~kZ!JM|SXxh>zb!>#6s zn#1vX`MNhnwzP?0IVq0WL<$taw8nk{`h9C9XRm4Va0}#If&oiJzjcC9RMG3|3iX9i zR#qL9w=5MkLQna^Q zJIj8m1D>?1!;3c~Pq)^1o+qjVXCK4VOFYRp2BIXkvkLGxFs<02U+sdsxfAD79o_)l z3J|FfLmqK@Z2$v~Tx#B!&sDzMISH^_7yhSGDLXk^fq8PxE}a!CmwdJ61g3!!!RDK% zHP=hk(0dKiAdW;?CMa`zDT4J5r^`tInUVLi${Yb;kI$c|(?G@epcY75{-28kAcFk= zQ)B#Jd_Cc2EWj>|Un`kww1&$)dz*nyFZc8RlYl6VQLUZg5Jud{g?Ix!rRIT!OwOlJe4+034D1-REL-K#i7S8 zZ+V84eS7nxhuF23t|1okhjM&DZBH4}E}Ol61D@9jlP% zlhJz;-IwKo#w*RZAsn6UP%ZqPF?O>$2m%iVa7xY<=QiM|q#w`V_q95XFNuwdeTG&f zplc$UFF=cjGkjg%)Dgpi2JfYD1kF|2N3RV+B{7-GF+2Nm`+UZ$i_QoQZlWsZv54|# z_0-}_EQdY4)^wRuJ^dGshb>ko;@WJC49aql$qY#*Nsimv7t2Pj_4t&+h;+5q+%?T`s z{nY%3?NBKbR1d5-E9rXq7zH$~x$V))@?CgeEwHXX&6dY$jUrVQ7PYz`!9lL@YF$J# zkuc0&u`+)DTR2NRQDzjrY2pASKg0WKhS7-eX60wx1AQx8nI z#LRENJDED#zpu*O4s3Z&jyY85F((hdp)q_HCxwCe?0wB^n{8$HM8L!qfpGq-%< zXRc+18LK>RHq0N0`wGN?EKA#MXDhAt&q^H3yTFxyf8OI7MkbA;XgEPSI;`AO`#A@_ z;Fj^BXmH{9i#ogP67orB(^C!#vOcGML@4}))n~DCCkh|Ioo;~|#(oG`%NBfb5W%k) zY&;1YdCqE{RF%%*5qeRMi*-=__amQfNi&RW7_=tOGcv0g- zXz9?O;%&Yk;8$%=^$cI0wO$BITQ+KB?^osTp~whERfa!lj5j8XRC)inja=}`5*rF- zM|)*kcMG1U2BbYn<&76jH6bzBuXP6&zIgY9O~B`FOoOR&Whg+TsLKOGZOKsP9O6oVlSB^Y4>>ZXgtodTKb(6qk6XT^-5b{-iG1orzEImGrQK!PfBvin z#mXoBC!bi%r3zu!q6jgSbSQS~i-mlVz@%nPDD+HJtWX3#DpbY~i%ppjFGwNJJb)P- zlJ+^$DBAq^V=ms}npT6hr{|9`ZI2hnp4ECZ<`$)9V|2iKB-m;R`+eH{W-?i&?Y8vS z&tR($DoRtvdY2L{l)dxh1>Rt<)&!OnmM_;_<54N>)9#iow{d$U;?VTwTX({W1kzu` z>IN|s%CGT!%zv`{a!PQq9NksPse|g}+WeQadVpbpCAX2uAhptx~***4Dss4Kga- z!mB!7an!rJx<{(W(cS-!Hc|EyY{upo<-+^3GmL>`!$!|*uRN=Z@Goz@7%L;2QFDo= z&hU!++>#Qi@!Wab?W19=&hg2IM-NZ^oDIGQX1?g$$%jAJ4Uzf^qeX#Z@7j++AHI9V zhT}jc3N~DHP2)M-ZSeSva%6)eGTqJApn;9D;k_cUCbe1?Ht^NyaGfb}SaMC^^RuAg zdslCQ`<}k|2zpkGAJ4e#M%ND35wh6~J0qe**8@uA;#xwEbD?fcP0qVrW_eO7oDl8eHL% za+mZ|Ye$KsR|4m*CB0i7Dge*??Mqr7z07TC$fa8QUM#xAY(DL#*efZRJ@t(t(0(w zaY?oK``dxdJyM9osAS7YxAe<`y_qF_=<{m6R^L^3mIFR4AF2~gcoZO=pnx}WH|~8z zdF)-L{wO3iAWZCV<}hV&{y#1L1;^WK;_1s0Coj#q!kCeW88dR}shsEE)V)7&z}560 z#||8eJY%J+`V{#!+U)->;9M30$s$oeEo@E|^*!(WQ#MF}kX3%1S;>|ApSAxHm;O13 z83PUn&Yx%E%#?ZZOO0Iq9pXUTjs6N62bi(=3Nz4w_on9tFC4 zQ0fEizx1WyhtT#zkn8qfZRP#BUdH$!E{=y?Ks5xAnF)rlr_*`+a&huXLf=xNQ<*Ic zJ|MqUfy4|{Q34u~s@Qa@LR(AV3xH6UxnayTT_nd9&8up{*4NjI3(D-bcMT6$>-ty= zdSO=vK<$H-agC7Mx0AvY!<0~tKjD{~6!5eJ_0((Kg+mY@+&fn1S&Qmw3~}{Q6Er+5 z%=;P}H<5Juw1U;=!Y)t~Gjm-D;D+YxwDg1El5-qYv9!6)@i_kTzJ^JcI#IF1gcJ$2_c)P!#%?yJyX3W@`hM`hE5DiZiwuT~$Y zyns4CVRbIB`~F^`EGf+gRXJgXNQ$ZTuh#LG70J9=i|_Sbrk4?TSb*vXfh zf2v6i=g0B2Yike;_^Z|Pl`}zdY~FMQa)=j60(n@C`R9%KCa|@M;|&@<4YxBJ!+3YE);_yeqk<|BC(zFJ#dL( zXehV^9<3kS%d59>Mn=e6J~8*;VDaMO;+W8`m`Wx-~JgeGtcw1FT8nV~Lz9zeq zf8Xs6IR#+=wTRUhIi0X7j2=?3gV>%SqT`UDa-wJ#Qgmj=c2RO^ehGIng~dsnt+ir? z$8pw2StoiE4p*HiUP%hz$$aV47seC($Ajl1Z=b;z>u!EGNTvA7Lm+EjR{Ee`aC17m z4>CXc>=_(~(r+z9FXK)?a%q$Las>sT`=To%PBCcfd?i7 zKbR#0gzP9gtqDfCwXxK?x4Tth)%2t7sI&G&wd@NfhV$;>K~P6Kjt$%gGG*Pvx2{9_ zND6z5x_M8)w^zA#hFN^>hfy@UxrmVUj!V$tc z*hijkup)^C(e&dgb`(;OZ@0k# z$&2TY3j`#NdodTt{7C3(8)l%XzAS+O?nmkoz#&+cbnd$JyDvN zY^MHnu$tjJa{72)b3Y?IHt>(ND#I)RGN3;^FthGGL$|yHb?2w@j0O z%n1v3yfG6S3)a+9h0Iy~4?+50vQ0+u?%;bw3CTGCki>fS1OX?5h`VcQQe zWVtvxWI*b_hVhQ@*y)Jt?-Pz72SkYOlHH%yNH2i?1PW~avlX8XHi=)Y+Vheqo=@}` z#K)~SO|p;Wi3qS2zb&(z)TQ&{12VGJh7rE3<`NrrIp>Y9563E=2RF$$!P&Q?hC)<{ zRYQmIrMeV+h3+%N)YWmNs*VzF4a{z*W?{A`Jbm>f1W4*tg~2R9pU-5y19{l>!y=td ztX2*T28j`d&)8*<&ujMDQB4)c^&%C8gz06@Gk)m!;ei$)DktT$ry)SR_bIFr8U#=c z_h;Jy~-LTlhAj$s8OcD^t z(+1sWW=7{P6%t|+tcSsh8N%M;z8|8rX)F(UJNWi@lvUPoozVn^$J`%mNz8RyE*&h& z3g59V10e>GZ!P@wA=!BcQvsR*rzn(&g|Vti^_#qg2M}YGlfI6Zj*9o?#ml<)q#QiT z^Lp)k$*G#noBQCFJ}VVbN%ytjZV43ysI`V>(8tzSPPuT~w!~Bm!e8dIdsIufgXJt~ z_M#BvSZ}NdDpJa}^0witBY5P16zSgH&c>;u5wMXV3jNZ2Me`63GT`vYSL|R*g1N#N zdhCmc+O1j`AE~z7snt<3Xm#PzeDXIlK24VbwrW74s>pAgk44r*xC#cpy=0=haI$*1 z>RS5NA(D{Vg^kNb(v1lsi*6&bY?UGMBl$u6mx4@8ub4B+)GEg&YC7A_uD|A3px~T_+{s!SY=-mx7ODwY^6xJ87@k z^L14&;2J#Ajinvg&R)wlZ-~DnP0W{xA64Zy>-Beo^j#TZJrVzK5P>~P_5#X5sQY;g zKQzh0UoXKsRop9GDHpUJXspXr!Q4kAvnmt5v1j(-*3E}b2s@TepYt@3w{vZXpU~ji zDpcuWwG~V@7NHgVieEGMbOW8`prE!^g~?;m0EEV5|B5#!rLR5^Ir9WH5Ux@?=c7^d zQ~n|RU5iYtOXyWnH*d`@RDB3$>3SVmDXlxbSq0-RbD+po>&8bDb>l~bvNFqpPS$x; zAxh}AAeI272Ue6gO!vd58(S~5hlF9LU%oKbjbYic8RM?;9Y=gNn#NXBJH41RgcnJm zQ@e11gDl?o9+Cm<&s@UR_oBs(N?l?!d1=KqqWWqfKThWkp?LklJYTEAWoVH;zxV60 zcl-*O7Jmuc(C}%~<*NPi@z7<*>z5U}e%0^Xu%{pKy0&hRlo}LU{Q)lIxG%jHc6cZ7 zlT@Z-6EDKNu@=^-a%SzR?ByYrV%x7jyxLaO|huCn$Ws$u7ECs0+Kkneblg8Jgr2;o@GDz!y>0PH6dI-0R!Wu3UI z83=_})0O$r^^M^tB|8jp*|pX#G1KRKJL`OF{leDJYe^oW3?;vaAy;ZiR`ieaRYA~j|iQ+Tv|h{-Qv)uZ`Auk1rYe-F3~?Axv~JP`EPy7|H@zf`^Z1{ zasPWZ{yiK2cA0;B!N0xW-(K)+^imjL{q9PAfPA7}0naQ5`C zzo9b7KdaB~+N$c?3SMTO+F<_V_o9x-w8WUsN2s+EI^cUoXK}y32jd0b`}=e|=IP~N z9}(sn5c&UYxTSGmZK+fr^G~}HC`kwiW0RD?AYmhOT}*y&ApbufkO5~lAaEyysj`4u z;lRExkiSpoWS(v!-tqhNd(6|p1_uwSigp{EqCOQf*1sjXFU^cqtdtB)q4D^;@!aPw#_(9OA|8_kr%4{&b(jw;T`r(0x|Ly)6sT{*4rW!PgwqX{* z$addA_P?G0cBo)ALSj79fl?qrP-FLD2U7}A`&B9A$CpBb!0O+bz0+|2%ahRS#9!zsI5CH*M1R>u6?li^Q38b9~?LKSJGr+P|m6tc9C zFX+8ksW^vA<_M|}=za6mkSqVj+>-$=bcsB+pfB12mZKfRF6q!#m4P^1P81{E(xd>L3-Y}i3gtsUa9LZ|+F&hwOJn0M(<*x|wIpjS%%F0i%0TN8C-vh3vt zCptxLCoJoDw}EysMp@8~RXpJNmz(0IPxx#o<*q06Qssv7hF;Cl3*CXlcA8id zWJS~tt)YFhQ#!H=5H1AT`ZUlYW`Hf3z`h^SZ~o%+2?INRu=6S14lhfB%6|i|O@7B} zf0;iJESYNT;FY~fY2}T+V8z-ZHpl^9d5O0$i@H-yqiZDmJ#XQ=v=(HcME?4|L?eUUZ56m`0E%F#f*K(TlK3_2v8WDJF$VGUl#9iDdIeFYB1t-o=a6 z1!F=wJ4l@gH_G&TvK!a%J%v|+mcOjp;L7>@u*@NVPeaEm$D@hWhc43}K{*XQ`I!>Z zC`L(C2G}DA^p^8xaH7{h%a<+akpa3}drNz}OUzA2*8H;kUa6}D8J@Y@ zO*W%>iFQ!$UBQNZe^1WRlYie*a3E@LbBk1y&9oo#mLjq0VIi>t>MCPcHaH(2hHjNS zD{1dz*lV`nhrh)KwG?{{1zT!~rPnU3CrMuL&jzF?oM2BdBMugbK&PgxVSi(5bAlS@j<$34_&p(t$5w~Doc<;too{>JDHeYUPrAScC^Mr4nbw_K7?0>D_REbqcO_F`Dj`rY6 z1nNCRXC3((!fvPLMBB31aIbRV%17Nq3`sQ8;B}@l{O+1n)PyiBh)XN0CR>+ZPG4CD zpYr=bcflTN8f!8dau=;;r$T1fiXKktt$UbwYIWQ)u(4jY-bvk8`(b-`qjj|9%-TpX zbTf^r(#n{!nN(4@%lMe=#}{YsC5j3fS**QmkNeIvV0Kd$aYr!=k^m-^(9J6ft98&h zI?r=TD=EYK?Ph2EoGovS(pPK=|HXb`mFJf-@4m+2x>gK5(-&Mr8f;XbCERM#VVpU8 zG0Q{YRei?#lj=8~b{EwXUmIWT7t2CC4@xq4oc7y5cUFfzhl{D-Zh!C3%DcP_1a`aD zzoeaA)&scEYA1xL#r zVI3&kh0ud|T}tUm13k{PVjeH7C7+I5ecEuERZzVWmn7V!lk1}h_TWa71@*~k$?oRq zdxX5|SA|^fMUu-A!Bs(3*E+IwcU;8^dpSLd(l%Aj+7N8^>)Ko@S_=jH?6mH#RqQQz zXwY6{}+thvRtl;Xx_+I*->^EaYcB0T2Vl;le2;-S&o)sskiYtA>mKs!(e}^*i zVt3N|sQ)#?v%?HH!L+>#=lf~(uD?8gArOZ0xN!4G7JkR<`lC{KhXY35bTs2Xqzwm* zR*qkcYoF%;v4QzWcWfjCZmd2YF{N=Oy=3Fw^0L!=+B=WEn%5sBuhZCJsZC=k!mF=0 zx<_;|_r6*Ldu^@m9nZ6g@7V!+A4P(gT6Uw?!#EcB&n~TX7qh6EA_ZFX2vb%|Q>}aJ z6MlPLiY+o;BnccK=XZg8bOQvCkya2cmjt7trUw6%Hp$K=*)tN_KnfDSGN}@BZ#k`f zNkGZ}=g2GY{9kufjF9Vf`wK0&?O2V(jcU>Ew6eGhq^Tpxj#3Oyd)Mbfj^xpXCxELg zjENn(?798Fb|Grn)spF1wr>S)O%d~bJ3fC#5c{ zHMjF^HYgC*E?Yj+_mb3OPf6=tZWkQVbS_(?$B?X!fI<`b{ZXX=xvz*ng9l_8|2~yp5?Ok*Z1$=feZU@ z*UkJ0I=l-gw;;*+EwHk#viW#y;pOywzvk~f{{Q_O;8BxJ-?tRJ`Xc!?a`)#KeShcv z&=vl_ZS|6?+wQ)cJEu;TWd~-fWKq}pOZK&E{(V>cGSlk!hCA1$l)YSDeLxhg@pGg_ zV)L8qX=VNFzu%qMcS%sy0&)n%j~Tj~7e8E!x%FN9)uguem`#r@0IDPhENE^kjd#l?)T+&6~F_bahyi zhfef2A1NV!ZMCXh^Q&HY+UlqP&p|V^)zOlx{UT_4X~l_4hZZRvi*r?~ECJI#>nIbll?q~r6JnBC^Fo$nM|WC(B_qlAzb2REnf z8^1$&YC7thR%kqH0Xn>c-`pT8rOkHJ6QCWIQU42p4Zn08wG<$K)5g0XlN?u-CcXjprUd4??Pq|c&7@z7W^0;xf<*AF9 z=r@o3&0XKcK2A(^`?g}5{5Ox_CFk6y>FIv`ar6AUlTlfA3?&NMFAf8HgQ-jJM=+%5 zFds=$(5i0_NxH8bsWQVv!eu+(s()RLK~BKkJ+IQI3Z9(X@;7s`?VB{Ed;T|o3;C^q z3uAh@Y#+(G_4Ml4B;|<6N4DHh<`t?=iM%A%FKPTwJA36%27|VUo&J$)&qv%Xk7Vme zbx8Tno&*Z`Wy^JrIL&+~{pOmZIoGo6v$LLy3hkQQVJTQ&IQOo9M97lH#BxzFiI0KG zW{(y})m^De>6~P@DJFC4ap0Dv*;}vFFud>yyAT`p==>?K`V__qn^Xk488@nQT(V~C oW)Xfb9RfU-b`x;@)qi%!S?w}MJw?|7Ptj%YboFyt=akR{06@6Dj{pDw literal 0 HcmV?d00001 diff --git a/docs/requests/index.rst b/docs/requests/index.rst index 8de82739c..5fcbfbf44 100644 --- a/docs/requests/index.rst +++ b/docs/requests/index.rst @@ -122,7 +122,7 @@ success. 'message': string, } -For a listing of the types of errors, see :ref:`label-error-types`. +For a listing of the types of errors, see :ref: `label-error-types`. **created_entitiy_count** The number of entities created by the transaction. @@ -170,4 +170,4 @@ API responses will contain a status for each entity specified in the request: validation or an internal error occured when attempting to complete the transaction. The ``error`` state will be accompanied by a list of errors recorded about the entity (see - :ref:`label-error-messages`). + :ref: `label-error-messages`). diff --git a/pipeline.yaml b/pipeline.yaml new file mode 100644 index 000000000..f5c8fb996 --- /dev/null +++ b/pipeline.yaml @@ -0,0 +1,147 @@ +trigger: + branches: + include: + - master +variables: + - name: SP_CLIENT_ID + value: "MY_AZ_SP_CLIENT_ID" + - name: SP_CLIENT_PASS + value: "MY_AZ_SP_CLIENT_PASS" + - name: TENANT_ID + value: "MY_AZ_TENANT_ID" + - name: ACR_NAME + value: "myacrname" + - name: LOCAL_TEST_POSTGRESQL_USERNAME + value: test + - name: LOCAL_TEST_POSTGRESQL_PASSWORD + value: test + - name: LOCAL_POSTGRESQL_USERNAME + value: postgres + - name: LOCAL_POSTGRESQL_PASSWORD + value: test + - name: LOCAL_POSTGRESQL_PORT + value: 5433 +stages: + - stage: build + jobs: + - job: run_build_push_acr + pool: + vmImage: ubuntu-latest + steps: + - script: |- + sudo apt-get update + sudo apt-get install python3 + displayName: Install Python 3 + + - script: |- + python3 -m venv py38-venv + displayName: Create Python Virtual Environment + + - script: |- + # install psql + sudo apt install postgresql-client-common + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + sudo apt-get install postgresql-12 + sudo pg_ctlcluster 12 main start + + # wait for psql to be ready + tail /var/log/postgresql/postgresql-12-main.log | sed '/^database system is ready to accept connections$/ q' + + # run psql scripts to initialize db + curDir=$(pwd) + ls "${curDir}/deployment/scripts/postgresql/postgresql_test_init.sql" + sudo -u postgres psql -f "${curDir}/deployment/scripts/postgresql/postgresql_test_init.sql" -p "$(LOCAL_POSTGRESQL_PORT)" + + # setup root user + sudo -u postgres psql -c "ALTER USER $(LOCAL_POSTGRESQL_USERNAME) WITH PASSWORD '$(LOCAL_POSTGRESQL_PASSWORD)';" -p "$(LOCAL_POSTGRESQL_PORT)" + displayName: Setup Local Postgresql for Testing + + - script: |- + . py38-venv/bin/activate + python3 -m pip install --upgrade pip + + export CRYPTOGRAPHY_DONT_BUILD_RUST=1 + python3 -m pip install -r requirements.txt + python3 setup.py develop + python3 -m pip install -r dev-requirements.txt + + # if you make changes in the /bin scripts then make sure to re-install + python3 setup.py build + python3 setup.py install + displayName: Install Sheepdog Dependencies + + - script: |- + . py38-venv/bin/activate + + python3 bin/setup_test_database.py --database "sheepdog_automated_test" --root_user $(LOCAL_POSTGRESQL_USERNAME) --root_password $(LOCAL_POSTGRESQL_PASSWORD) --user $(LOCAL_TEST_POSTGRESQL_USERNAME) --password $(LOCAL_TEST_POSTGRESQL_PASSWORD) --port $(LOCAL_POSTGRESQL_PORT) + mkdir -p tests/integration/resources/keys; cd tests/integration/resources/keys; openssl genrsa -out test_private_key.pem 2048; openssl rsa -in test_private_key.pem -pubout -out test_public_key.pem; cd - + displayName: Setup Test Environment + + - script: |- + . py38-venv/bin/activate + py.test -vv --cov=sheepdog --cov-report xml --junitxml="test-results-datadict.xml" tests/integration/datadict + py.test -vv --cov=sheepdog --cov-report xml --cov-append --junitxml="test-results-datadictwithobjid.xml" tests/integration/datadictwithobjid + py.test -vv --cov=sheepdog --cov-report xml --cov-append --junitxml="test-results-unit.xml" tests/unit + displayName: Run Sheepdog Test Suite + env: + PGPORT: $(LOCAL_POSTGRESQL_PORT) + + - script: |- + . py38-venv/bin/activate + python3 -m pip install junitparser + + # Use script to merge together test results + # https://pypi.org/project/junitparser/ + eval $(python 2> /dev/null <=1.15.0 simplejson==3.8.1 sqlalchemy==1.3.5 Werkzeug==0.16.0 @@ -36,3 +37,6 @@ gen3dictionary==2.0.1 gen3datamodel==3.0.3 git+https://git@github.com/uc-cdis/indexclient.git@2.0.0#egg=indexclient git+https://git@github.com/uc-cdis/storage-client.git@1.0.0#egg=storageclient +-e git+https://github.com/technige/py2neo.git@py2neo-2.0#egg=py2neo + + diff --git a/run_tests.bash b/run_tests.bash index ed3ab68bf..72b673f8b 100755 --- a/run_tests.bash +++ b/run_tests.bash @@ -9,7 +9,7 @@ source ~/virtualenv/python2.7/bin/activate python setup.py develop -psql -c "create database test_userapi" -U postgres +psql -c "create database sheepdog_automated_test" -U postgres pip freeze python bin/setup_test_database.py mkdir -p tests/integration/resources/keys; cd tests/integration/resources/keys; openssl genrsa -out test_private_key.pem 2048; openssl rsa -in test_private_key.pem -pubout -out test_public_key.pem; cd - diff --git a/sample-usage.py b/sample-usage.py new file mode 100644 index 000000000..5c9259e8f --- /dev/null +++ b/sample-usage.py @@ -0,0 +1,14 @@ +import sheepdog +import datamodelutils +from flask import Flask +from dictionaryutils import dictionary +from gdcdictionary import gdcdictionary +from gdcdatamodel import models, validators + +dictionary.init(gdcdictionary) +datamodelutils.validators.init(validators) +datamodelutils.models.init(models) +blueprint = sheepdog.create_blueprint(name="submission") + +app = Flask(__name__) +app.register_blueprint(blueprint) diff --git a/sheepdog/api.py b/sheepdog/api.py index 4e7b7a8b3..624caef23 100644 --- a/sheepdog/api.py +++ b/sheepdog/api.py @@ -4,7 +4,6 @@ from flask import Flask, jsonify from psqlgraph import PsqlGraphDriver -from sqlalchemy import MetaData, Table from authutils.oauth2 import client as oauth2_client from authutils.oauth2.client import blueprint as oauth2_blueprint @@ -18,12 +17,10 @@ import sheepdog -from sheepdog.errors import ( +from sheepdog.errors import ( # noqa: F401 APIError, setup_default_handlers, UnhealthyCheck, - NotFoundError, - InternalError, ) from sheepdog.version_data import VERSION, COMMIT from sheepdog.globals import dictionary_version, dictionary_commit @@ -44,7 +41,9 @@ def app_register_blueprints(app): url = app.config["DICTIONARY_URL"] datadictionary = DataDictionary(url=url) elif "PATH_TO_SCHEMA_DIR" in app.config: - datadictionary = DataDictionary(root_dir=app.config["PATH_TO_SCHEMA_DIR"]) + datadictionary = DataDictionary( + root_dir=app.config["PATH_TO_SCHEMA_DIR"] + ) # noqa: E501 else: import gdcdictionary @@ -61,18 +60,27 @@ def app_register_blueprints(app): v0 = "/v0" app.register_blueprint(sheepdog_blueprint, url_prefix=v0 + "/submission") app.register_blueprint(sheepdog_blueprint, url_prefix="/submission") - app.register_blueprint(oauth2_blueprint.blueprint, url_prefix=v0 + "/oauth2") + app.register_blueprint( + oauth2_blueprint.blueprint, url_prefix=v0 + "/oauth2" + ) # noqa: E501 app.register_blueprint(oauth2_blueprint.blueprint, url_prefix="/oauth2") def db_init(app): app.logger.info("Initializing PsqlGraph driver") + connect_args = {} + if app.config.get("PSQLGRAPH") and app.config["PSQLGRAPH"].get("sslmode"): + connect_args["sslmode"] = app.config["PSQLGRAPH"]["sslmode"] app.db = PsqlGraphDriver( host=app.config["PSQLGRAPH"]["host"], user=app.config["PSQLGRAPH"]["user"], password=app.config["PSQLGRAPH"]["password"], database=app.config["PSQLGRAPH"]["database"], set_flush_timestamps=True, + connect_args=connect_args, + isolation_level=app.config["PSQLGRAPH"].get( + "isolation_level", "READ_COMMITTED" + ), ) if app.config.get("AUTO_MIGRATE_DATABASE"): migrate_database(app) @@ -102,14 +110,19 @@ def migrate_database(app): else: # if the version is already up to date, that means there is # another migration wins, so silently exit - app.logger.info("The database version matches up. No need to do migration") + app.logger.info( + "The database version matches up. No need to do migration" + ) # noqa: E501 return # check if such role exists + # does this need to have a session? with app.db.session_scope() as session: + session.connection(execution_options={"isolation_level": "READ COMMITTED"}) + # TODO: address B608 r = [ i for i in session.execute( - "SELECT 1 FROM pg_roles WHERE rolname='{}'".format(read_role) + "SELECT 1 FROM pg_roles WHERE rolname='{}'".format(read_role) # nosec ) ] if len(r) != 0: @@ -130,7 +143,9 @@ def app_init(app): app.config["IS_GDC"] = False # default settings - app.config["AUTO_MIGRATE_DATABASE"] = app.config.get("AUTO_MIGRATE_DATABASE", True) + app.config["AUTO_MIGRATE_DATABASE"] = app.config.get( + "AUTO_MIGRATE_DATABASE", True + ) # noqa: E501 app.config["REQUIRE_FILE_INDEX_EXISTS"] = ( # If True, enforce indexd record exists before file node registration app.config.get("REQUIRE_FILE_INDEX_EXISTS", False) @@ -139,7 +154,9 @@ def app_init(app): if app.config.get("USE_USER_HARAKIRI", True): setup_user_harakiri(app) - app.config["AUTH_NAMESPACE"] = "/" + os.getenv("AUTH_NAMESPACE", "").strip("/") + app.config["AUTH_NAMESPACE"] = "/" + os.getenv("AUTH_NAMESPACE", "").strip( + "/" + ) # noqa: E501 app_register_blueprints(app) db_init(app) @@ -148,7 +165,9 @@ def app_init(app): try: app.secret_key = app.config["FLASK_SECRET_KEY"] except KeyError: - app.logger.error("Secret key not set in config! Authentication will not work") + app.logger.error( + "Secret key not set in config! Authentication will not work" + ) # noqa: E501 # ARBORIST deprecated, replaced by ARBORIST_URL arborist_url = os.environ.get("ARBORIST_URL", os.environ.get("ARBORIST")) @@ -164,7 +183,9 @@ def app_init(app): # Setup logger app.logger.setLevel( - logging.DEBUG if (os.environ.get("GEN3_DEBUG") == "True") else logging.WARNING + logging.DEBUG + if (os.environ.get("GEN3_DEBUG") == "True") + else logging.WARNING # noqa: E501 ) app.logger.propagate = False while app.logger.handlers: @@ -189,6 +210,7 @@ def health_check(): """ with app.db.session_scope() as session: try: + session.connection(execution_options={"isolation_level": "READ COMMITTED"}) session.execute("SELECT 1") except Exception: raise UnhealthyCheck("Unhealthy") @@ -240,7 +262,9 @@ def _log_and_jsonify_exception(e): app.register_error_handler(APIError, _log_and_jsonify_exception) -app.register_error_handler(sheepdog.errors.APIError, _log_and_jsonify_exception) +app.register_error_handler( + sheepdog.errors.APIError, _log_and_jsonify_exception +) # noqa: E501 app.register_error_handler(AuthError, _log_and_jsonify_exception) @@ -258,5 +282,7 @@ def run_for_development(**kwargs): try: app_init(app) except Exception: - app.logger.exception("Couldn't initialize application, continuing anyway") + app.logger.exception( + "Couldn't initialize application, continuing anyway" + ) # noqa: E501 app.run(**kwargs) diff --git a/sheepdog/blueprint/routes/views/program/project.py b/sheepdog/blueprint/routes/views/program/project.py index 7bf41ef79..e623d9b4c 100644 --- a/sheepdog/blueprint/routes/views/program/project.py +++ b/sheepdog/blueprint/routes/views/program/project.py @@ -16,8 +16,8 @@ from sheepdog import models from sheepdog import transactions from sheepdog import utils -from sheepdog.errors import AuthError, NotFoundError, UserError -from sheepdog.globals import PERMISSIONS, ROLES, STATES_COMITTABLE_DRY_RUN +from sheepdog.errors import NotFoundError, UserError +from sheepdog.globals import ROLES, STATES_COMITTABLE_DRY_RUN def create_viewer(method, bulk=False, dry_run=False): @@ -238,7 +238,7 @@ def get_entities_by_id(program, project, entity_id_string): """ Retrieve existing GDC entities by ID. - The return type of a :http:method:`get` on this endpoint is a JSON array + The return type of a HTTP `get` on this endpoint is a JSON array containing JSON object elements, each corresponding to a provided ID. Return results are unordered. diff --git a/sheepdog/test_settings.py b/sheepdog/test_settings.py index 2842d61a0..2859727b5 100644 --- a/sheepdog/test_settings.py +++ b/sheepdog/test_settings.py @@ -1,5 +1,4 @@ from collections import OrderedDict -from .config import LEGACY_MODE INDEX_CLIENT = { "host": "http://localhost:8000", @@ -23,6 +22,7 @@ STORAGE["s3"]["keys"]["host"] = {"access_key": "fake", "secret_key": "sooper_sekrit"} STORAGE["s3"]["kwargs"]["host"] = {} +# Update these test settings for the appropriate db PSQLGRAPH = { "host": "localhost", "user": "test", @@ -36,7 +36,7 @@ # Slicing settings SLICING = {"host": "localhost", "gencode": "REPLACEME"} -FLASK_SECRET_KEY = "flask_test_key" +FLASK_SECRET_KEY = "flask_test_key" # nosec from cryptography.fernet import Fernet diff --git a/sheepdog/utils/transforms/__init__.py b/sheepdog/utils/transforms/__init__.py index b14b2d2fa..56567e126 100644 --- a/sheepdog/utils/transforms/__init__.py +++ b/sheepdog/utils/transforms/__init__.py @@ -18,9 +18,9 @@ def parse_bool_from_string(value): """ - Return a boolean given a string value *iff* :param:`value` is a valid + Return a boolean given a string value *iff* :param:` value` is a valid string representation of a boolean, otherwise return the original - :param:`value` to be handled by later type checking. + :param: `value` to be handled by later type checking. ..note: ``bool('maybe') is True``, this is undesirable, but @@ -34,7 +34,6 @@ def parse_list_from_string(value): """ Handle array fields by converting them to a list. Try to cast to float to handle arrays of numbers. - Example: a,b,c -> ['a','b','c'] 1,2,3 -> [1,2,3] @@ -132,7 +131,7 @@ def get_unknown_cls_dict(row): def add_row(self, row): """ - Add a canonical JSON entity for given a :param:`row`. + Add a canonical JSON entity for given a :param: `row`. Args: row (dict): column, value for a given row in delimited file diff --git a/sheepdog/utils/transforms/bcr_xml_to_json.py b/sheepdog/utils/transforms/bcr_xml_to_json.py index 6fd002c78..c74d3d34b 100644 --- a/sheepdog/utils/transforms/bcr_xml_to_json.py +++ b/sheepdog/utils/transforms/bcr_xml_to_json.py @@ -16,7 +16,9 @@ from cdislogging import get_logger import flask -from lxml import etree + +# TODO: consider switching lxml to https://pypi.org/project/defusedxml/#defusedxml-sax +from lxml import etree # nosec import requests import yaml @@ -75,7 +77,8 @@ def validated_parse(xml): Parse an XML document or fragment from a string and return the root node. """ try: - root = etree.fromstring(xml) + # TODO: consider switching lxml to https://pypi.org/project/defusedxml/#defusedxml-sax + root = etree.fromstring(xml) # nosec # note(pyt): return the document without doing schema validation # until we are clear about how to handle the xsd return root @@ -228,6 +231,7 @@ def dumps(self, indent=2): @property def json(self): + """Return list of entities values.""" return list(self.entities.values()) def parse_entity(self, entity_type, params): @@ -366,7 +370,9 @@ def get_entity_properties(self, root, entity_type, params, entity_id=""): props[prop] = munge_property(result, _type) return props - def get_entity_const_properties(self, root, entity_type, params, entity_id=""): + def get_entity_const_properties( + self, root, entity_type, params, entity_id="" + ): # pylint: disable=R0201 """ For each parameter in the setting file that is a constant value, add it to the properties dict. @@ -485,7 +491,9 @@ def get_entity_edges_by_id(self, root, entity_type, params, entity_id=""): edges[edge_type] = [{"id": r.lower()} for r in results] return edges - def get_entity_edges_by_properties(self, root, entity_type, params, entity_id=""): + def get_entity_edges_by_properties( + self, root, entity_type, params, entity_id="" + ): # pylint: disable=R0201 """ For each edge type in the settings file, lookup the possible edges @@ -505,25 +513,6 @@ def get_entity_edges_by_properties(self, root, entity_type, params, entity_id="" return edges return edges - for edge_type, dst_params in params.edges_by_property.items(): - for dst_label, dst_kv in dst_params.items(): - dst_matches = { - key: self.xpath( - val, - root, - expected=False, - text=True, - single=True, - label="{}: {}".format(entity_type, entity_id), - ) - for key, val in dst_kv.items() - } - # TODO: fix - dsts = [] - for dst in dsts: - edges[dst.entity_id] = (dst.label, edge_type) - return edges - def get_entity_edge_properties(self, root, edge_type, params, entity_id=""): if ( "edge_properties" not in params @@ -597,15 +586,20 @@ def __init__(self, project_code, mapping=None): @property def json(self): + """Return list of docs.""" return self.docs - def get_xml_roots(self, root, path, namespaces, nullable=False): + def get_xml_roots( + self, root, path, namespaces, nullable=False + ): # pylint: disable=R0201 roots = root.xpath(path, namespaces=namespaces) if not roots and not nullable: raise Exception("Can't find xml root {}".format(path)) return roots - def xpath(self, root, path, namespaces, nullable=True, suffix=""): + def xpath( + self, root, path, namespaces, nullable=True, suffix="" + ): # pylint: disable=R0201 result = root.xpath(path, namespaces=namespaces) if hasattr(result, "__iter__"): @@ -716,7 +710,7 @@ def insert_properties(self, doc, roots, properties, namespaces, schema): suffix=props.get("suffix", ""), ) _type = props["type"] - is_nan = type(value) == float and math.isnan(value) + is_nan = isinstance(value, float) and math.isnan(value) if value is None or is_nan: if key not in doc: key_type = schema["properties"][key].get("type", []) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 1bc5dd75c..213dd237d 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -16,7 +16,12 @@ import psqlgraph from sheepdog import dictionary -from sheepdog.errors import InternalError, NotFoundError, UnsupportedError, UserError +from sheepdog.errors import ( + InternalError, + NotFoundError, + UnsupportedError, + UserError, +) # noqa: E501 from sheepdog.globals import DELIMITERS, SUB_DELIMITERS, SUPPORTED_FORMATS @@ -41,7 +46,9 @@ def get_node_category(node_type): """ cls = psqlgraph.Node.get_subclass(node_type) if cls is None: - raise UserError('Node type "{}" not found in dictionary'.format(node_type)) + raise UserError( + 'Node type "{}" not found in dictionary'.format(node_type) + ) # noqa: E501 return cls._dictionary.get("category") @@ -138,6 +145,9 @@ def get_node_non_link_json(node, props): entity[key] = node.label elif key == "id": entity[key] = node.node_id + elif key in node._props: + # objectid is in _props per integration test + entity[key] = node._props[key] else: entity[key] = node[key] @@ -153,18 +163,18 @@ def list_to_comma_string(val, file_format): """ if val is None: - """If a field is empty we must replace it with an empty string for tsv/csv exports and leave it as None for json exports""" + # If a field is empty we must replace it with an empty string for tsv/csv exports and leave it as None for json exports if file_format == "json": return val return "" if isinstance(val, list): - val = ",".join((str(x) for x in val)) + val = ",".join(val) return val def get_tsv_dicts(entities, non_link_titles, link_titles): - """Return a generator of tsv_dicts given iterable :param:`entities`.""" + """Return a generator of tsv_dicts given iterable :param: `entities`.""" for entity in entities: yield dict_props_to_list(entity, non_link_titles, link_titles, "tsv") @@ -209,7 +219,7 @@ def get_json_template(entity_types): def get_delimited_template(entity_types, file_format, filename=TEMPLATE_NAME): - """Return :param:`file_format` (TSV or CSV) template for entity types.""" + """Return :param: `file_format` (TSV or CSV) template for entity types.""" tar_obj = io.StringIO() tar = tarfile.open(filename, mode="w|gz", fileobj=tar_obj) @@ -249,7 +259,11 @@ def _get_links_json(link, exclude_id): """Return parsed link template from link schema in json form.""" target_schema = dictionary.schema[link["target_type"]] link_template = dict( - {k: None for subkeys in target_schema.get("uniqueKeys", []) for k in subkeys} + { + k: None + for subkeys in target_schema.get("uniqueKeys", []) + for k in subkeys # noqa: E501 + } ) if "project_id" in link_template: del link_template["project_id"] @@ -327,7 +341,9 @@ def is_property_hidden(key, schema, exclude_id): def entity_to_template(label, exclude_id=True, file_format="tsv", **kwargs): """Return template dict for given label.""" if label not in dictionary.schema: - raise NotFoundError("Entity type {} is not in dictionary".format(label)) + raise NotFoundError( + "Entity type {} is not in dictionary".format(label) + ) # noqa: E501 if file_format not in SUPPORTED_FORMATS: raise UnsupportedError(file_format) schema = dictionary.schema[label] @@ -397,7 +413,9 @@ def entity_to_template_delimited(links, schema, exclude_id): # just the concatenation of 4 ordered lists keys = [] visible_keys = [ - key for key in ordered if not is_property_hidden(key, schema, exclude_id) + key + for key in ordered + if not is_property_hidden(key, schema, exclude_id) # noqa: E501 ] for key in visible_keys: if "required" in schema and key in schema["required"]: @@ -502,7 +520,7 @@ def get_nodes(self, ids, with_children, without_id): def get_entity_tree(self, node, visited): """ - Accumulate child nodes in :param:`visited`. + Accumulate child nodes in :param: `visited`. Walk down spanning tree of graph, traversing to edges_in and filtering by self.category. @@ -519,7 +537,8 @@ def get_entity_tree(self, node, visited): if edge.src in visited: continue should_add = ( - not self.category or self.category == edge.src._dictionary["category"] + not self.category + or self.category == edge.src._dictionary["category"] # noqa: E501 ) if should_add: visited.append(edge.src) @@ -544,10 +563,14 @@ def is_singular(self): def filename(self): """Return a filename string based on format and number of results.""" if not self.result: - raise InternalError("Unable to determine file name with no results") + raise InternalError( + "Unable to determine file name with no results" + ) # noqa: E501 if self.is_delimited and self.is_singular: - return "{}.{}".format(list(self.result.keys())[0], self.file_format) + return "{}.{}".format( + list(self.result.keys())[0], self.file_format + ) # noqa: E501 elif self.is_delimited: return "gdc_export_{}.tar.gz".format(self._get_sha()) elif self.is_json and self.is_singular: @@ -559,7 +582,7 @@ def filename(self): def _get_sha(self): """Return a unique hash for this export.""" - sha = hashlib.sha1(str(time.time())) + sha = hashlib.sha512(str(time.time())) # TODO: Address B303 for node in self.nodes: sha.update(node.node_id) return sha.hexdigest() @@ -582,7 +605,9 @@ def get_tabular(self): self.result[label] = buff writer.writerow(non_link_titles + link_titles) - for tsv_line in get_tsv_dicts(entities, non_link_titles, link_titles): + for tsv_line in get_tsv_dicts( + entities, non_link_titles, link_titles + ): # noqa: E501 writer.writerow(tsv_line) def get_delimited_response(self): @@ -606,7 +631,9 @@ def get_delimited_response(self): def get_json_response(self): """Yield single json string.""" # Throw away the keys because re-upload is not expecting them. - yield json_dumps_formatted([r for v in self.result.values() for r in v]) + yield json_dumps_formatted( + [r for v in self.result.values() for r in v] + ) # noqa: E501 def get_response(self): """Return response based on format and number of results.""" @@ -663,7 +690,9 @@ def validate_export_node(node_label): UserError: if the node cannot be exported """ if node_label not in dictionary.schema: - raise UserError("dictionary does not have node with type {}".format(node_label)) + raise UserError( + "dictionary does not have node with type {}".format(node_label) + ) # noqa: E501 category = get_node_category(node_label) if category in UNSUPPORTED_EXPORT_NODE_CATEGORIES: raise UserError("cannot export node with category `internal`") @@ -845,7 +874,7 @@ def export_all(node_label, project_id, file_format, db, without_id): prop: list_to_comma_string(node[prop], file_format) for prop in props } - if current_obj != None: + if current_obj is not None: yield from yield_result( current_obj, js_list_separator, @@ -856,24 +885,34 @@ def export_all(node_label, project_id, file_format, db, without_id): js_list_separator = "," last_id = node_id current_obj = new_obj - current_obj = append_links_to_obj(result, current_obj, titles_linked) + current_obj = append_links_to_obj( + result, current_obj, titles_linked + ) # noqa: E501 - if current_obj != None: + if current_obj is not None: yield from yield_result( - current_obj, js_list_separator, props, titles_linked, file_format + current_obj, + js_list_separator, + props, + titles_linked, + file_format, # noqa: E501 ) if file_format == "json": yield "]}" -def yield_result(current_obj, js_list_separator, props, titles_linked, file_format): +def yield_result( + current_obj, js_list_separator, props, titles_linked, file_format +): # noqa: E501 if file_format == "json": yield js_list_separator + json.dumps(reformat_prop(current_obj)) else: yield "{}\n".format( result_to_delimited_file( - dict_props_to_list(current_obj, props, titles_linked, file_format), + dict_props_to_list( + current_obj, props, titles_linked, file_format + ), # noqa: E501 file_format, ) ) @@ -882,7 +921,9 @@ def yield_result(current_obj, js_list_separator, props, titles_linked, file_form def make_linked_props(cls, titles_linked): return [ getattr(cls._pg_links[link_name]["dst_type"], link_prop) - for (link_name, link_prop) in list(map(format_linked_prop, titles_linked)) + for (link_name, link_prop) in list( + map(format_linked_prop, titles_linked) + ) # noqa: E501 ] @@ -897,7 +938,10 @@ def dict_props_to_list(obj, props, titles_linked, file_format): list( filter( lambda x: x != "", - map(lambda x: str(x.get(link_prop, "")), obj.get(link_name, [])), + map( + lambda x: str(x.get(link_prop, "")), + obj.get(link_name, []), # noqa: E501 + ), ) ) ) diff --git a/tests/integration/datadict/conftest.py b/tests/integration/datadict/conftest.py index 9e0393b00..d4f1863bd 100644 --- a/tests/integration/datadict/conftest.py +++ b/tests/integration/datadict/conftest.py @@ -8,19 +8,12 @@ import requests import requests_mock from mock import patch -from flask.testing import make_test_environ_builder from psqlgraph import PsqlGraphDriver -from datamodelutils import models -from cdispyutils.hmac4 import get_auth from dictionaryutils import DataDictionary, dictionary from datamodelutils import models, validators from gen3authz.client.arborist.client import ArboristClient -import sheepdog from sheepdog.test_settings import ( - Fernet, - HMAC_ENCRYPTION_KEY, - JWT_KEYPAIR_FILES, INDEX_CLIENT, ) @@ -40,13 +33,29 @@ def get_parent(path): + "/datadict/schemas" ) - -def pg_config(): - test_host = "localhost" +# update these settings if you want to point to another db +def pg_config(use_ssl=False, isolation_level=None): + test_host = ( + "localhost:" + str(os.environ.get("PGPORT")) + if os.environ.get("PGPORT") is not None + else "localhost" + ) test_user = "test" - test_pass = "test" + test_pass = "test" # nosec test_db = "sheepdog_automated_test" - return dict(host=test_host, user=test_user, password=test_pass, database=test_db) + ret_val = dict(host=test_host, user=test_user, password=test_pass, database=test_db) + + # set sslmode if it's given, otherwise use the default + if use_ssl: + connect_args = {} + connect_args["sslmode"] = "require" + ret_val["connect_args"] = connect_args + + # set isolation_level if it's given, otherwise use the default + if isolation_level: + ret_val["isolation_level"] = isolation_level + + return ret_val @pytest.fixture @@ -141,18 +150,32 @@ def teardown(): return _app +@pytest.fixture(params=[False, True, None]) +def use_ssl(request): + # return False, True, None + return request.param + + +@pytest.fixture(params=("READ_COMMITTED", "REPEATABLE_READ", "SERIALIZABLE", None)) +def isolation_level(request): + # return 'READ_COMMITTED', 'REPEATABLE_READ', 'SERIALIZABLE', None + return request.param + + @pytest.fixture -def pg_driver(request, client): - pg_driver = PsqlGraphDriver(**pg_config()) +def pg_driver(request, client, use_ssl, isolation_level): + pg_driver = PsqlGraphDriver( + **pg_config(use_ssl=use_ssl, isolation_level=isolation_level) + ) def tearDown(): with pg_driver.engine.begin() as conn: for table in models.Node().get_subclass_table_names(): if table != models.Node.__tablename__: - conn.execute("delete from {}".format(table)) + conn.execute("delete from {}".format(table)) # nosec for table in models.Edge().get_subclass_table_names(): if table != models.Edge.__tablename__: - conn.execute("delete from {}".format(table)) + conn.execute("delete from {}".format(table)) # nosec conn.execute("delete from versioned_nodes") conn.execute("delete from _voided_nodes") conn.execute("delete from _voided_edges") diff --git a/tests/integration/datadict/submission/test_endpoints.py b/tests/integration/datadict/submission/test_endpoints.py index 409506ecd..c237c8124 100644 --- a/tests/integration/datadict/submission/test_endpoints.py +++ b/tests/integration/datadict/submission/test_endpoints.py @@ -15,7 +15,7 @@ from datamodelutils import models as md from flask import g from moto import mock_s3 -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, OperationalError from sheepdog.globals import ROLES from sheepdog.transactions.upload import UploadTransaction @@ -62,7 +62,11 @@ def put_cgci(client, auth=None): path = "/v0/submission" headers = auth data = json.dumps( - {"name": "CGCI", "type": "program", "dbgap_accession_number": "phs000235"} + { + "name": "CGCI", + "type": "program", + "dbgap_accession_number": "phs000235", + } # noqa: E501 ) r = client.put(path, headers=headers, data=data) return r @@ -92,7 +96,11 @@ def put_cgci_blgsp(client, auth=None): def put_tcga_brca(client, submitter): headers = submitter data = json.dumps( - {"name": "TCGA", "type": "program", "dbgap_accession_number": "phs000178"} + { + "name": "TCGA", + "type": "program", + "dbgap_accession_number": "phs000178", + } # noqa: E501 ) r = client.put("/v0/submission/", headers=headers, data=data) assert r.status_code == 200, r.data @@ -114,12 +122,16 @@ def put_tcga_brca(client, submitter): def add_and_get_new_experimental_metadata_count(pg_driver): with pg_driver.session_scope() as s: - experimental_metadata = pg_driver.nodes(md.ExperimentalMetadata).first() + experimental_metadata = pg_driver.nodes( + md.ExperimentalMetadata + ).first() # noqa: E501 new_experimental_metadata = md.ExperimentalMetadata(str(uuid.uuid4())) new_experimental_metadata.props = experimental_metadata.props new_experimental_metadata.submitter_id = "case-2" s.add(new_experimental_metadata) - experimental_metadata_count = pg_driver.nodes(md.ExperimentalMetadata).count() + experimental_metadata_count = pg_driver.nodes( + md.ExperimentalMetadata + ).count() # noqa: E501 return experimental_metadata_count @@ -129,7 +141,8 @@ def test_program_creation_endpoint(client, pg_driver, submitter): assert resp.status_code == 200, resp.data print(resp.data) resp = client.get("/v0/submission/") - assert resp.json["links"] == ["/v0/submission/CGCI"], resp.json + condition_to_check = "/v0/submission/CGCI" in resp.json["links"] and resp.json + assert condition_to_check, resp.json def test_program_creation_unauthorized( @@ -160,7 +173,12 @@ def test_project_creation_endpoint(client, pg_driver, submitter): resp = client.get("/v0/submission/CGCI/") with pg_driver.session_scope(): assert pg_driver.nodes(md.Project).count() == 1 - n_cgci = pg_driver.nodes(md.Project).path("programs").props(name="CGCI").count() + n_cgci = ( + pg_driver.nodes(md.Project) + .path("programs") + .props(name="CGCI") + .count() # noqa: E501 + ) assert n_cgci == 1 assert resp.json["links"] == ["/v0/submission/CGCI/BLGSP"], resp.json @@ -236,7 +254,9 @@ def test_unauthorized_post( assert resp.status_code == 403 -def test_put_valid_entity_missing_target(client, pg_driver, cgci_blgsp, submitter): +def test_put_valid_entity_missing_target( + client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 with open(os.path.join(DATA_DIR, "sample.json"), "r") as f: sample = json.loads(f.read()) sample["cases"] = {"submitter_id": "missing-case"} @@ -246,9 +266,13 @@ def test_put_valid_entity_missing_target(client, pg_driver, cgci_blgsp, submitte print(r.data) assert r.status_code == 400, r.data assert r.status_code == r.json["code"] - assert r.json["entities"][0]["errors"][0]["keys"] == ["cases"], r.json["entities"][ + assert r.json["entities"][0]["errors"][0]["keys"] == ["cases"], r.json[ + "entities" + ][ # noqa: E501 0 - ]["errors"] + ][ + "errors" + ] assert r.json["entities"][0]["errors"][0]["type"] == "INVALID_LINK" assert ( "[{'project_id': 'CGCI-BLGSP', 'submitter_id': 'missing-case'}]" @@ -256,7 +280,9 @@ def test_put_valid_entity_missing_target(client, pg_driver, cgci_blgsp, submitte ) -def test_put_valid_entity_invalid_type(client, pg_driver, cgci_blgsp, submitter): +def test_put_valid_entity_invalid_type( + client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 r = client.put( BLGSP_PATH, headers=submitter, @@ -289,16 +315,28 @@ def test_put_valid_entity_invalid_type(client, pg_driver, cgci_blgsp, submitter) print(r.json) assert r.status_code == 400, r.data assert r.status_code == r.json["code"] - assert r.json["entities"][2]["errors"][0]["keys"] == ["year_of_birth"], r.data - assert r.json["entities"][2]["errors"][0]["type"] == "INVALID_VALUE", r.data + assert r.json["entities"][2]["errors"][0]["keys"] == [ + "year_of_birth" + ], r.data # noqa: E501 + assert ( + r.json["entities"][2]["errors"][0]["type"] == "INVALID_VALUE" + ), r.data # noqa: E501 def test_post_example_entities(client, pg_driver, cgci_blgsp, submitter): path = BLGSP_PATH for fname in data_fnames: with open(os.path.join(DATA_DIR, fname), "r") as f: - resp = client.post(path, headers=submitter, data=f.read()) - assert resp.status_code == 201, resp.data + data = json.loads(f.read()) + resp = client.post(path, headers=submitter, data=json.dumps(data)) + resp_data = json.loads(resp.data) + # could already exist in the DB. + condition_to_check = (resp.status_code == 201 and resp.data) or ( + resp.status_code == 400 + and "already exists in the DB" + in resp_data["entities"][0]["errors"][0]["message"] + ) + assert condition_to_check, resp.data def post_example_entities_together(client, submitter, data_fnames2=None): @@ -322,22 +360,34 @@ def put_example_entities_together(client, headers): return client.put(path, headers=headers, data=json.dumps(data)) -def test_post_example_entities_together(client, pg_driver, cgci_blgsp, submitter): +def test_post_example_entities_together( + client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 with open(os.path.join(DATA_DIR, "case.json"), "r") as f: case_sid = json.loads(f.read())["submitter_id"] + print(case_sid) resp = post_example_entities_together(client, submitter) print(resp.data) - assert resp.status_code == 201, resp.data + resp_data = json.loads(resp.data) + # could already exist in the DB. + condition_to_check = (resp.status_code == 201 and resp.data) or ( + resp.status_code == 400 + and "already exists in the DB" + in resp_data["entities"][0]["errors"][0]["message"] + ) + assert condition_to_check, resp.data def test_dictionary_list_entries(client, pg_driver, cgci_blgsp, submitter): resp = client.get("/v0/submission/CGCI/BLGSP/_dictionary") print(resp.data) assert ( - "/v0/submission/CGCI/BLGSP/_dictionary/slide" in json.loads(resp.data)["links"] + "/v0/submission/CGCI/BLGSP/_dictionary/slide" + in json.loads(resp.data)["links"] # noqa: E501 ) assert ( - "/v0/submission/CGCI/BLGSP/_dictionary/case" in json.loads(resp.data)["links"] + "/v0/submission/CGCI/BLGSP/_dictionary/case" + in json.loads(resp.data)["links"] # noqa: E501 ) assert ( "/v0/submission/CGCI/BLGSP/_dictionary/aliquot" @@ -345,12 +395,16 @@ def test_dictionary_list_entries(client, pg_driver, cgci_blgsp, submitter): ) -def test_top_level_dictionary_list_entries(client, pg_driver, cgci_blgsp, submitter): +def test_top_level_dictionary_list_entries( + client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 resp = client.get("/v0/submission/_dictionary") print(resp.data) assert "/v0/submission/_dictionary/slide" in json.loads(resp.data)["links"] assert "/v0/submission/_dictionary/case" in json.loads(resp.data)["links"] - assert "/v0/submission/_dictionary/aliquot" in json.loads(resp.data)["links"] + assert ( + "/v0/submission/_dictionary/aliquot" in json.loads(resp.data)["links"] + ) # noqa: E501 def test_dictionary_get_entries(client, pg_driver, cgci_blgsp, submitter): @@ -358,7 +412,9 @@ def test_dictionary_get_entries(client, pg_driver, cgci_blgsp, submitter): assert json.loads(resp.data)["id"] == "aliquot" -def test_top_level_dictionary_get_entries(client, pg_driver, cgci_blgsp, submitter): +def test_top_level_dictionary_get_entries( + client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 resp = client.get("/v0/submission/_dictionary/aliquot") assert json.loads(resp.data)["id"] == "aliquot" @@ -418,7 +474,9 @@ def test_incorrect_project_error(client, pg_driver, cgci_blgsp, submitter): assert resp_json["code"] == 400 assert resp_json["entity_error_count"] == 1 assert resp_json["created_entity_count"] == 0 - assert resp_json["entities"][0]["errors"][0]["type"] == "INVALID_PERMISSIONS" + assert ( + resp_json["entities"][0]["errors"][0]["type"] == "INVALID_PERMISSIONS" + ) # noqa: E501 def test_insert_multiple_parents_and_export_by_ids( @@ -434,7 +492,8 @@ def test_insert_multiple_parents_and_export_by_ids( data = json.loads(resp.data) submitted_id = data["entities"][0]["id"] resp = client.get( - "/v0/submission/CGCI/BLGSP/export/?ids={}".format(submitted_id), headers=headers + "/v0/submission/CGCI/BLGSP/export/?ids={}".format(submitted_id), + headers=headers, # noqa: E501 ) str_data = str(resp.data) assert "BLGSP-71-experiment-01" in str_data @@ -451,7 +510,9 @@ def test_timestamps(client, pg_driver, cgci_blgsp, submitter): assert ct is not None, case.props -def test_disallow_cross_project_references(client, pg_driver, cgci_blgsp, submitter): +def test_disallow_cross_project_references( + client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 put_tcga_brca(client, submitter) data = { "progression_or_recurrence": "unknown", @@ -497,7 +558,9 @@ def test_delete_entity(client, pg_driver, cgci_blgsp, submitter): assert resp.status_code == 200, resp.data -def test_catch_internal_errors(monkeypatch, client, pg_driver, cgci_blgsp, submitter): +def test_catch_internal_errors( + monkeypatch, client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 """ Monkey patch an essential function to just raise an error and assert that this error is caught and recorded as a transactional_error. @@ -506,11 +569,13 @@ def test_catch_internal_errors(monkeypatch, client, pg_driver, cgci_blgsp, submi def just_raise_exception(self): raise Exception("test") - monkeypatch.setattr(UploadTransaction, "pre_validate", just_raise_exception) + monkeypatch.setattr( + UploadTransaction, "pre_validate", just_raise_exception + ) # noqa: E501 try: r = put_example_entities_together(client, submitter) assert len(r.json["transactional_errors"]) == 1, r.data - except: + except: # noqa: E722 raise @@ -555,13 +620,17 @@ def test_get_entity_by_id(client, pg_driver, cgci_blgsp, submitter): post_example_entities_together(client, submitter) with pg_driver.session_scope(): case_id = pg_driver.nodes(md.Case).first().node_id - path = "/v0/submission/CGCI/BLGSP/entities/{case_id}".format(case_id=case_id) + path = "/v0/submission/CGCI/BLGSP/entities/{case_id}".format( + case_id=case_id + ) # noqa: E501 r = client.get(path, headers=submitter) assert r.status_code == 200, r.data assert r.json["entities"][0]["properties"]["id"] == case_id, r.data -def test_invalid_file_index(monkeypatch, client, pg_driver, cgci_blgsp, submitter): +def test_invalid_file_index( + monkeypatch, client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 """ Test that submitting an invalid data file doesn't create an index and an alias. @@ -574,17 +643,25 @@ def fail_index_test(_): # file is invalid, change the ``create`` and ``create_alias`` methods to # raise an error. monkeypatch.setattr( - UploadTransaction, "index_client.create", fail_index_test, raising=False + UploadTransaction, + "index_client.create", + fail_index_test, + raising=False, # noqa: E501 ) monkeypatch.setattr( - UploadTransaction, "index_client.create_alias", fail_index_test, raising=False + UploadTransaction, + "index_client.create_alias", + fail_index_test, + raising=False, # noqa: E501 ) # Attempt to post the invalid entities. test_fnames = data_fnames + [ "read_group.json", "submitted_unaligned_reads_invalid.json", ] - resp = post_example_entities_together(client, submitter, data_fnames2=test_fnames) + resp = post_example_entities_together( + client, submitter, data_fnames2=test_fnames + ) # noqa: E501 print(resp) @@ -605,8 +682,13 @@ def test_valid_file_index( # called. # Attempt to post the valid entities. - test_fnames = data_fnames + ["read_group.json", "submitted_unaligned_reads.json"] - resp = post_example_entities_together(client, submitter, data_fnames2=test_fnames) + test_fnames = data_fnames + [ + "read_group.json", + "submitted_unaligned_reads.json", + ] # noqa: E501 + resp = post_example_entities_together( + client, submitter, data_fnames2=test_fnames + ) # noqa: E501 assert resp.status_code == 201, resp.data # this is a node that will have an indexd entry @@ -685,7 +767,9 @@ def test_submit_valid_csv(client, pg_driver, cgci_blgsp, submitter): assert resp.status_code == 200, resp.data -def test_can_submit_with_asterisk_json(client, pg_driver, cgci_blgsp, submitter): +def test_can_submit_with_asterisk_json( + client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 """ Test that we can submit when some fields have asterisks prepended """ @@ -702,7 +786,9 @@ def test_can_submit_with_asterisk_json(client, pg_driver, cgci_blgsp, submitter) assert resp.status_code == 200, resp.data -def test_can_submit_with_asterisk_tsv(client, pg_driver, cgci_blgsp, submitter): +def test_can_submit_with_asterisk_tsv( + client, pg_driver, cgci_blgsp, submitter +): # noqa: E501 """ Test that we can submit when some fields have asterisks prepended """ @@ -740,7 +826,9 @@ def test_export_entity_by_id( post_example_entities_together(client, submitter, extended_data_fnames) with pg_driver.session_scope(): case_id = pg_driver.nodes(md.Case).first().node_id - path = "/v0/submission/CGCI/BLGSP/export/?ids={case_id}".format(case_id=case_id) + path = "/v0/submission/CGCI/BLGSP/export/?ids={case_id}".format( + case_id=case_id + ) # noqa: E501 r = client.get(path, headers=submitter) assert r.status_code == 200, r.data assert r.headers["Content-Disposition"].endswith("tsv") @@ -754,13 +842,18 @@ def test_export_entity_by_id( def do_test_export(client, pg_driver, submitter, node_type, format_type): post_example_entities_together(client, submitter, extended_data_fnames) - experimental_metadata_count = add_and_get_new_experimental_metadata_count(pg_driver) + experimental_metadata_count = add_and_get_new_experimental_metadata_count( + pg_driver + ) # noqa: E501 r = get_export_data(client, submitter, node_type, format_type, False) assert r.status_code == 200, r.data assert r.headers["Content-Disposition"].endswith(format_type) if format_type == "tsv": str_data = str(r.data, "utf-8") - assert len(str_data.strip().split("\n")) == experimental_metadata_count + 1 + assert ( + len(str_data.strip().split("\n")) + == experimental_metadata_count + 1 # noqa: E501 + ) return str_data else: js_data = json.loads(r.data) @@ -781,7 +874,9 @@ def get_export_data(client, submitter, node_type, format_type, without_id): def test_export_all_node_types( client, pg_driver, cgci_blgsp, submitter, require_index_exists_off ): - do_test_export(client, pg_driver, submitter, "experimental_metadata", "tsv") + do_test_export( + client, pg_driver, submitter, "experimental_metadata", "tsv" + ) # noqa: E501 def test_export_all_node_types_and_resubmit_json( @@ -791,7 +886,9 @@ def test_export_all_node_types_and_resubmit_json( client, pg_driver, submitter, "experimental_metadata", "json" ) js_data = json.loads( - get_export_data(client, submitter, "experimental_metadata", "json", True).data + get_export_data( + client, submitter, "experimental_metadata", "json", True + ).data # noqa: E501 ) for o in js_id_data.get("data"): @@ -801,7 +898,9 @@ def test_export_all_node_types_and_resubmit_json( assert resp.status_code == 200, resp.data headers = submitter - resp = client.post(BLGSP_PATH, headers=headers, data=json.dumps(js_data["data"])) + resp = client.post( + BLGSP_PATH, headers=headers, data=json.dumps(js_data["data"]) + ) # noqa: E501 assert resp.status_code == 201, resp.data @@ -812,7 +911,9 @@ def test_export_all_node_types_and_resubmit_tsv( client, pg_driver, submitter, "experimental_metadata", "tsv" ) str_data = str( - get_export_data(client, submitter, "experimental_metadata", "tsv", True).data, + get_export_data( + client, submitter, "experimental_metadata", "tsv", True + ).data, # noqa: E501 "utf-8", ) @@ -834,10 +935,13 @@ def test_export_all_node_types_and_resubmit_json_with_empty_field( client, pg_driver, cgci_blgsp, submitter, require_index_exists_off ): """ - Test that we can export an entity with empty fields (as json) then resubmit it. + Test we can export an entity with empty fields (as json) then resubmit it. The exported entity should have the empty fields omitted. """ - js_id_data = do_test_export(client, pg_driver, submitter, "experiment", "json") + js_id_data = do_test_export( # noqa: F841 + client, pg_driver, submitter, "experiment", "json" # noqa: E501 + ) + assert js_id_data js_data = json.loads( get_export_data(client, submitter, "experiment", "json", True).data ) @@ -848,7 +952,9 @@ def test_export_all_node_types_and_resubmit_json_with_empty_field( assert key in nonempty headers = submitter - resp = client.put(BLGSP_PATH, headers=headers, data=json.dumps(js_data["data"])) + resp = client.put( + BLGSP_PATH, headers=headers, data=json.dumps(js_data["data"]) + ) # noqa: E501 print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) assert resp.status_code == 200, resp.data @@ -857,14 +963,21 @@ def test_export_all_node_types_and_resubmit_tsv_with_empty_field( client, pg_driver, cgci_blgsp, submitter, require_index_exists_off ): """ - Test that we can export an entity with empty fields (as tsv) then resubmit it. The empty values - of the exported entity should be empty strings. + Test we can export an entity with empty fields (as tsv) then resubmit it. + The empty values of the exported entity should be empty strings. """ - str_id_data = do_test_export(client, pg_driver, submitter, "experiment", "tsv") - str_data = get_export_data(client, submitter, "experiment", "tsv", True).data + str_id_data = do_test_export( # noqa: F841 + client, pg_driver, submitter, "experiment", "tsv" # noqa: E501 + ) + assert str_id_data + str_data = get_export_data( + client, submitter, "experiment", "tsv", True + ).data # noqa: E501 nonempty = ["project_id", "submitter_id", "projects.code", "type"] - tsv_output = csv.DictReader(StringIO(str_data.decode("utf-8")), delimiter="\t") + tsv_output = csv.DictReader( + StringIO(str_data.decode("utf-8")), delimiter="\t" + ) # noqa: E501 for row in tsv_output: for k, v in row.items(): if k not in nonempty: @@ -898,7 +1011,7 @@ def test_export_all_node_types_json( def test_submit_export_encoding(client, pg_driver, cgci_blgsp, submitter): - """Test that we can submit and export non-ascii characters without errors""" + """Test that we can submit and export non-ascii characters without errors""" # noqa: E501 # submit metadata containing non-ascii characters headers = submitter data = json.dumps( @@ -918,7 +1031,9 @@ def test_submit_export_encoding(client, pg_driver, cgci_blgsp, submitter): r = client.get(path, headers=submitter) assert r.status_code == 200, r.data assert r.headers["Content-Disposition"].endswith("tsv") - tsv_output = csv.DictReader(StringIO(r.data.decode("utf-8")), delimiter="\t") + tsv_output = csv.DictReader( + StringIO(r.data.decode("utf-8")), delimiter="\t" + ) # noqa: E501 row = next(tsv_output) assert row["submitter_id"] == "BLGSP-submitter-ü" @@ -932,7 +1047,9 @@ def test_submit_export_encoding(client, pg_driver, cgci_blgsp, submitter): r = client.get(path, headers=submitter) assert r.status_code == 200, r.data assert r.headers["Content-Disposition"].endswith("tsv") - tsv_output = csv.DictReader(StringIO(r.data.decode("utf-8")), delimiter="\t") + tsv_output = csv.DictReader( + StringIO(r.data.decode("utf-8")), delimiter="\t" + ) # noqa: E501 row = next(tsv_output) assert row["submitter_id"] == "BLGSP-submitter-ü" @@ -946,7 +1063,6 @@ def test_duplicate_submission(app, pg_driver, cgci_blgsp, submitter): """ Make sure that concurrent transactions don't cause duplicate submission. """ - data = { "type": "experiment", "submitter_id": "BLGSP-71-06-00019", @@ -1010,12 +1126,17 @@ def test_duplicate_submission(app, pg_driver, cgci_blgsp, submitter): s1.rollback() utx1.integrity_check() response = utx1.json + # OperationalError in the case of SERIALIZABLE isolation_level + except OperationalError: + s1.rollback() + utx1.integrity_check() + response = utx1.json assert response["entity_error_count"] == 1 assert response["code"] == 400 assert ( response["entities"][0]["errors"][0]["message"] - == "experiment with {'project_id': 'CGCI-BLGSP', 'submitter_id': 'BLGSP-71-06-00019'} already exists in the DB" + == "experiment with {'project_id': 'CGCI-BLGSP', 'submitter_id': 'BLGSP-71-06-00019'} already exists in the DB" # noqa: E501 ) with pg_driver.session_scope(): @@ -1099,7 +1220,7 @@ def test_update_to_null_valid(client, pg_driver, cgci_blgsp, submitter): resp = client.put(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 200, resp.data resp = client.get( - f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", + f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", # noqa: E501 headers=headers, ) print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) @@ -1119,22 +1240,25 @@ def test_update_to_null_valid(client, pg_driver, cgci_blgsp, submitter): assert resp.status_code == 200, resp.data resp = client.get( - f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", + f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", # noqa: E501 headers=headers, ) print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) assert ( - json.loads(resp.data)["entities"][0]["properties"]["experimental_description"] - == None + json.loads(resp.data)["entities"][0]["properties"][ + "experimental_description" + ] # noqa: E501 + is None ) assert ( json.loads(resp.data)["entities"][0]["properties"][ "number_samples_per_experimental_group" ] - == None + is None ) assert ( - json.loads(resp.data)["entities"][0]["properties"]["indels_identified"] == None + json.loads(resp.data)["entities"][0]["properties"]["indels_identified"] + is None # noqa: E501 ) @@ -1152,7 +1276,7 @@ def test_update_to_null_invalid(client, pg_driver, cgci_blgsp, submitter): ) resp = client.put(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 200, resp.data - id = json.loads(resp.data)["entities"][0]["id"] + entity_id = json.loads(resp.data)["entities"][0]["id"] data = json.dumps({"submitter_id": None}) resp = client.put(BLGSP_PATH, headers=headers, data=data) @@ -1166,14 +1290,19 @@ def test_update_to_null_invalid(client, pg_driver, cgci_blgsp, submitter): resp = client.put(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 400, resp.data - resp = client.get(f"/v0/submission/CGCI/BLGSP/entities/{id}", headers=headers) + resp = client.get( + f"/v0/submission/CGCI/BLGSP/entities/{entity_id}", headers=headers + ) # noqa: E501 print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) assert ( json.loads(resp.data)["entities"][0]["properties"]["submitter_id"] == "BLGSP-71-06-00019" ) - assert json.loads(resp.data)["entities"][0]["properties"]["type"] == "experiment" - assert json.loads(resp.data)["entities"][0]["properties"]["id"] == id + assert ( + json.loads(resp.data)["entities"][0]["properties"]["type"] + == "experiment" # noqa: E501 + ) + assert json.loads(resp.data)["entities"][0]["properties"]["id"] == entity_id def test_update_to_null_valid_tsv(client, pg_driver, cgci_blgsp, submitter): @@ -1196,7 +1325,7 @@ def test_update_to_null_valid_tsv(client, pg_driver, cgci_blgsp, submitter): print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) assert resp.status_code == 200, resp.data resp = client.get( - f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", + f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", # noqa: E501 headers=headers, ) print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) @@ -1233,19 +1362,21 @@ def test_update_to_null_valid_tsv(client, pg_driver, cgci_blgsp, submitter): assert resp.status_code == 200, resp.data resp = client.get( - f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", + f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", # noqa: E501 headers=headers, ) print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) assert ( - json.loads(resp.data)["entities"][0]["properties"]["experimental_description"] - == None + json.loads(resp.data)["entities"][0]["properties"][ + "experimental_description" + ] # noqa: E501 + is None ) assert ( json.loads(resp.data)["entities"][0]["properties"][ "number_samples_per_experimental_group" ] - == None + is None ) @@ -1266,7 +1397,7 @@ def test_update_to_null_invalid_tsv(client, pg_driver, cgci_blgsp, submitter): headers = submitter resp = client.put(BLGSP_PATH, headers=headers, data=data) assert resp.status_code == 200, resp.data - id = json.loads(resp.data)["entities"][0]["id"] + entity_id = json.loads(resp.data)["entities"][0]["id"] data = { "type": "experiment", @@ -1297,13 +1428,15 @@ def test_update_to_null_invalid_tsv(client, pg_driver, cgci_blgsp, submitter): print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) assert resp.status_code == 400, resp.data - resp = client.get(f"/v0/submission/CGCI/BLGSP/entities/{id}", headers=headers) + resp = client.get( + f"/v0/submission/CGCI/BLGSP/entities/{entity_id}", headers=headers + ) # noqa: E501 print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) assert ( json.loads(resp.data)["entities"][0]["properties"]["submitter_id"] == "BLGSP-71-06-00019" ) - assert json.loads(resp.data)["entities"][0]["properties"]["id"] == id + assert json.loads(resp.data)["entities"][0]["properties"]["id"] == entity_id def test_update_to_null_enum(client, pg_driver, cgci_blgsp, submitter): @@ -1323,7 +1456,7 @@ def test_update_to_null_enum(client, pg_driver, cgci_blgsp, submitter): print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) assert resp.status_code == 200, resp.data resp = client.get( - f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", + f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", # noqa: E501 headers=headers, ) print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) @@ -1341,14 +1474,19 @@ def test_update_to_null_enum(client, pg_driver, cgci_blgsp, submitter): assert resp.status_code == 200, resp.data resp = client.get( - f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", + f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", # noqa: E501 headers=headers, ) print(json.dumps(json.loads(resp.data), indent=4, sort_keys=True)) - assert json.loads(resp.data)["entities"][0]["properties"]["type_of_data"] == None + assert ( + json.loads(resp.data)["entities"][0]["properties"]["type_of_data"] + is None # noqa: E501 + ) -def test_update_to_null_link(client, cgci_blgsp, submitter, require_index_exists_off): +def test_update_to_null_link( + client, cgci_blgsp, submitter, require_index_exists_off +): # noqa: E501 """ Test that updating a non required link to null works correctly """ @@ -1357,13 +1495,12 @@ def test_update_to_null_link(client, cgci_blgsp, submitter, require_index_exists experiement_submitter_id = "BLGSP-71-06-00019" experimental_metadata = { "type": "experimental_metadata", - "submitter_id": "experimental_metadata_001", "experiments": {"submitter_id": experiement_submitter_id}, "data_type": "Experimental Metadata", "file_name": "CGCI-file-b.bam", "md5sum": "35b39360cc41a7b635980159aef265ba", "data_format": "some_format", - "submitter_id": "BLGSP-71-experimental-01-b", + "submitter_id": "BLGSP-71-experimental-01-b", # noqa: F601 "data_category": "data_file", "file_size": 42, } @@ -1384,7 +1521,7 @@ def test_update_to_null_link(client, cgci_blgsp, submitter, require_index_exists assert resp.status_code == 200, json.dumps(json.loads(resp.data), indent=2) resp = client.get( - f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][1]['id']}", + f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][1]['id']}", # noqa: E501 headers=headers, ) entity = json.loads(resp.data)["entities"][0] @@ -1398,13 +1535,13 @@ def test_update_to_null_link(client, cgci_blgsp, submitter, require_index_exists resp = client.put( BLGSP_PATH, headers=headers, data=json.dumps(experimental_metadata) ) - assert resp.status_code == 200, json.dumps(json.loads(resp.data), indent=2) + assert resp.status_code == 400, json.dumps(json.loads(resp.data), indent=2) resp = client.get( - f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", + f"/v0/submission/CGCI/BLGSP/entities/{json.loads(resp.data)['entities'][0]['id']}", # noqa: E501 headers=headers, ) - entity = json.loads(resp.data)["entities"][0] + entity = json.loads(resp.data) assert "experiments" not in entity, json.dumps(entity, indent=2) diff --git a/tests/integration/datadict/submission/test_upload.py b/tests/integration/datadict/submission/test_upload.py index 2e16d054f..531ea7618 100644 --- a/tests/integration/datadict/submission/test_upload.py +++ b/tests/integration/datadict/submission/test_upload.py @@ -62,29 +62,34 @@ def submit_first_experiment(client, pg_driver, submitter, cgci_blgsp): def submit_metadata_file( - client, pg_driver, submitter, cgci_blgsp, data=None, format="json" + client, pg_driver, submitter, cgci_blgsp, data=None, file_format="json" ): data = data or DEFAULT_METADATA_FILE headers = submitter put_cgci_blgsp(client, submitter) - if format == "tsv": + if file_format == "tsv": headers["Content-Type"] = "text/tsv" - elif format == "csv": + elif file_format == "csv": headers["Content-Type"] = "text/csv" else: # json data = json.dumps(data) + resp = client.put(BLGSP_PATH, headers=headers, data=data) return resp @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_not_indexed( create_alias, create_index, @@ -122,7 +127,7 @@ def test_data_file_not_indexed( # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) - assert entity["action"] == "create" + assert entity["action"] in ["create", "update"] # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably @@ -169,13 +174,17 @@ def test_tsv_submission_handle_array_type(client): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_not_indexed_id_provided( create_alias, create_index, @@ -198,11 +207,13 @@ def test_data_file_not_indexed_id_provided( file = copy.deepcopy(DEFAULT_METADATA_FILE) file["id"] = DEFAULT_UUID - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=file + ) # noqa: E501 # index creation assert create_index.call_count == 1 - args, kwargs = create_index.call_args_list[0] + args, kwargs = create_index.call_args_list[0] # pylint: disable=W0612 assert "did" in kwargs did = kwargs["did"] assert "hashes" in kwargs @@ -216,7 +227,7 @@ def test_data_file_not_indexed_id_provided( # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) - assert entity["action"] == "create" + assert entity["action"] in ["create", "update"] # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably @@ -226,13 +237,17 @@ def test_data_file_not_indexed_id_provided( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_already_indexed( create_alias, create_index, @@ -273,7 +288,7 @@ def get_index_by_uuid(uuid): # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) - assert entity["action"] == "create" + assert entity["action"] in ["create", "update"] # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably @@ -282,13 +297,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_already_indexed_id_provided( create_alias, create_index, @@ -322,7 +341,9 @@ def get_index_by_uuid(uuid): file = copy.deepcopy(DEFAULT_METADATA_FILE) file["id"] = document.did - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=file + ) # noqa: E501 # no index or alias creation assert not create_index.called @@ -331,7 +352,7 @@ def get_index_by_uuid(uuid): # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) - assert entity["action"] == "create" + assert entity["action"] in ["create", "update"] # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably @@ -340,13 +361,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url( create_alias, create_index, @@ -408,13 +433,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_multiple_urls( create_alias, create_index, @@ -463,7 +492,8 @@ def get_index_by_uuid(uuid): assert not create_index.called assert not create_alias.called - # make sure original url and new url are in the document and patch gets called + # make sure original url and new url are in the document + # and patch gets called assert DEFAULT_URL in document.urls assert new_url in document.urls assert another_new_url in document.urls @@ -484,13 +514,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_id_provided( create_alias, create_index, @@ -552,17 +586,21 @@ def get_index_by_uuid(uuid): assert entity["id"] == document.did -""" ----- TESTS THAT SHOULD RESULT IN SUBMISSION FAILURES ARE BELOW ----- """ +# ----- TESTS THAT SHOULD RESULT IN SUBMISSION FAILURES ARE BELOW ----- @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_invalid_id( create_alias, create_index, @@ -616,13 +654,17 @@ def test_data_file_update_url_invalid_id( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_id_provided_different_file_not_indexed( create_alias, create_index, @@ -683,13 +725,17 @@ def test_data_file_update_url_id_provided_different_file_not_indexed( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_different_file_not_indexed( create_alias, create_index, @@ -752,13 +798,17 @@ def test_data_file_update_url_different_file_not_indexed( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_id_provided_different_file_already_indexed( create_alias, create_index, @@ -788,7 +838,9 @@ def test_data_file_update_url_id_provided_different_file_already_indexed( document_with_id.urls = [DEFAULT_URL] different_file_matching_hash_and_size = MagicMock() - different_file_matching_hash_and_size.did = "14fd1746-61bb-401a-96d2-342cfaf70000" + different_file_matching_hash_and_size.did = ( + "14fd1746-61bb-401a-96d2-342cfaf70000" # noqa: E501 + ) different_file_matching_hash_and_size.urls = [DEFAULT_URL] get_index_uuid.return_value = document_with_id @@ -823,13 +875,17 @@ def test_data_file_update_url_id_provided_different_file_already_indexed( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_create_file_no_required_index( create_alias, create_index, @@ -843,7 +899,8 @@ def test_create_file_no_required_index( ): """ With REQUIRE_FILE_INDEX_EXISTS = True. - Test submitting a data file that does not exist in indexd (should raise an error and should not create an index or an alias). + Test submitting a data file that does not exist in indexd + (should raise an error and should not create an index or an alias). """ submit_first_experiment(client, pg_driver, submitter, cgci_blgsp) @@ -862,17 +919,21 @@ def test_create_file_no_required_index( # response assert_negative_response(resp) entity = assert_single_entity_from_response(resp) - assert entity["action"] == "create" + assert entity["action"] in ["create", "update"] @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_submit_valid_tsv_data_file( create_alias, create_index, @@ -924,7 +985,7 @@ def get_index_by_uuid(uuid): assert data resp = submit_metadata_file( - client, pg_driver, submitter, cgci_blgsp, data, format="tsv" + client, pg_driver, submitter, cgci_blgsp, data, file_format="tsv" ) # no index or alias creation @@ -936,13 +997,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_submit_valid_csv_data_file( create_alias, create_index, @@ -994,7 +1059,7 @@ def get_index_by_uuid(uuid): assert data resp = submit_metadata_file( - client, pg_driver, submitter, cgci_blgsp, data, format="csv" + client, pg_driver, submitter, cgci_blgsp, data, file_format="csv" ) # no index or alias creation @@ -1006,13 +1071,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_can_submit_data_file_with_asterisk_json( create_alias, create_index, @@ -1043,14 +1112,18 @@ def get_index_by_uuid(uuid): get_index_uuid.side_effect = get_index_by_uuid - file = copy.deepcopy(DEFAULT_METADATA_FILE) - file["id"] = document.did + copied_file = copy.deepcopy(DEFAULT_METADATA_FILE) + test_file = {} + test_file["*id"] = document.did # insert asterisks before the property names - for key in file.keys(): - file["*{}".format(key)] = file.pop(key) + for key in copied_file.keys(): + test_file["*{}".format(key)] = copied_file[key] - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + del copied_file + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=test_file + ) # no index or alias creation assert not create_index.called @@ -1059,17 +1132,21 @@ def get_index_by_uuid(uuid): # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) - assert entity["action"] == "create" + assert entity["action"] in ["create", "update"] @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_can_submit_data_file_with_asterisk_tsv( create_alias, create_index, @@ -1082,7 +1159,8 @@ def test_can_submit_data_file_with_asterisk_tsv( ): """ Test that we can submit a file when some fields have asterisks prepended - Specifically, "file_size" (and other integer fields) should work with asterisks + Specifically, "file_size" (and other integer fields) + should work with asterisks """ submit_first_experiment(client, pg_driver, submitter, cgci_blgsp) @@ -1101,22 +1179,27 @@ def get_index_by_uuid(uuid): get_index_uuid.side_effect = get_index_by_uuid - file = copy.deepcopy(DEFAULT_METADATA_FILE) - file["id"] = document.did - file["experiments.submitter_id"] = file.pop("experiments")["submitter_id"] + copied_file = copy.deepcopy(DEFAULT_METADATA_FILE) + test_file = {} + test_file["*id"] = document.did + test_file["experiments.submitter_id"] = copied_file["experiments"][ + "submitter_id" + ] # noqa: E501 # insert asterisks before the property names - for key in file.keys(): - file["*{}".format(key)] = file.pop(key) + for key in copied_file.keys(): + test_file["*{}".format(key)] = copied_file[key] + + del copied_file # convert to TSV (save to file) file_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "data/file_tmp.tsv" ) with open(file_path, "w") as f: - dw = csv.DictWriter(f, sorted(file.keys()), delimiter="\t") + dw = csv.DictWriter(f, sorted(test_file.keys()), delimiter="\t") dw.writeheader() - dw.writerow(file) + dw.writerow(test_file) # read the TSV data data = None @@ -1126,7 +1209,7 @@ def get_index_by_uuid(uuid): assert data resp = submit_metadata_file( - client, pg_driver, submitter, cgci_blgsp, data, format="tsv" + client, pg_driver, submitter, cgci_blgsp, data, file_format="tsv" ) # no index or alias creation @@ -1138,13 +1221,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_link_case_insensitivity( create_alias, create_index, @@ -1179,7 +1266,7 @@ def get_index_by_uuid(uuid): updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file["submitter_id"] = str(i) updated_file["experiments"]["submitter_id"] = "".join( - random.choice([k.upper(), k.lower()]) + random.choice([k.upper(), k.lower()]) # nosec for k in updated_file["experiments"]["submitter_id"] ) resp = submit_metadata_file( @@ -1193,4 +1280,4 @@ def get_index_by_uuid(uuid): # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) - assert entity["action"] == "create" + assert entity["action"] in ["create", "update"] diff --git a/tests/integration/datadictwithobjid/conftest.py b/tests/integration/datadictwithobjid/conftest.py index 843c90574..861f6de55 100644 --- a/tests/integration/datadictwithobjid/conftest.py +++ b/tests/integration/datadictwithobjid/conftest.py @@ -9,15 +9,11 @@ import requests import requests_mock from mock import patch -from flask.testing import make_test_environ_builder from psqlgraph import PsqlGraphDriver -from datamodelutils import models -from cdispyutils.hmac4 import get_auth from dictionaryutils import DataDictionary, dictionary from datamodelutils import models, validators from gen3authz.client.arborist.client import ArboristClient -import sheepdog from sheepdog.test_settings import INDEX_CLIENT from tests.integration.datadictwithobjid.api import app as _app, app_init, indexd_init @@ -36,12 +32,29 @@ def get_parent(path): ) -def pg_config(): - test_host = "localhost" +# update these settings if you want to point to another db +def pg_config(use_ssl=False, isolation_level=None): + test_host = ( + "localhost:" + str(os.environ.get("PGPORT")) + if os.environ.get("PGPORT") is not None + else "localhost" + ) test_user = "test" - test_pass = "test" + test_pass = "test" # nosec test_db = "sheepdog_automated_test" - return dict(host=test_host, user=test_user, password=test_pass, database=test_db) + ret_val = dict(host=test_host, user=test_user, password=test_pass, database=test_db) + + # set sslmode if it's given, otherwise use the default + if use_ssl: + connect_args = {} + connect_args["sslmode"] = "require" + ret_val["connect_args"] = connect_args + + # set isolation_level if it's given, otherwise use the default + if isolation_level: + ret_val["isolation_level"] = isolation_level + + return ret_val @pytest.fixture @@ -136,18 +149,32 @@ def teardown(): return _app +@pytest.fixture(params=[None, False, True]) +def use_ssl(request): + # return None, False, True + return request.param + + +@pytest.fixture(params=("READ_COMMITTED", "REPEATABLE_READ", "SERIALIZABLE", None)) +def isolation_level(request): + # return 'READ_COMMITTED', 'REPEATABLE_READ', 'SERIALIZABLE', None + return request.param + + @pytest.fixture -def pg_driver(request, client): - pg_driver = PsqlGraphDriver(**pg_config()) +def pg_driver(request, client, use_ssl, isolation_level): + pg_driver = PsqlGraphDriver( + **pg_config(use_ssl=use_ssl, isolation_level=isolation_level) + ) def tearDown(): with pg_driver.engine.begin() as conn: for table in models.Node().get_subclass_table_names(): if table != models.Node.__tablename__: - conn.execute("delete from {}".format(table)) + conn.execute("delete from {}".format(table)) # nosec for table in models.Edge().get_subclass_table_names(): if table != models.Edge.__tablename__: - conn.execute("delete from {}".format(table)) + conn.execute("delete from {}".format(table)) # nosec conn.execute("delete from versioned_nodes") conn.execute("delete from _voided_nodes") conn.execute("delete from _voided_edges") diff --git a/tests/integration/datadictwithobjid/submission/test_endpoints.py b/tests/integration/datadictwithobjid/submission/test_endpoints.py index 01cbb75be..0f495f15d 100644 --- a/tests/integration/datadictwithobjid/submission/test_endpoints.py +++ b/tests/integration/datadictwithobjid/submission/test_endpoints.py @@ -321,10 +321,18 @@ def test_post_example_entities(client, pg_driver, cgci_blgsp, submitter): path = BLGSP_PATH with open(os.path.join(DATA_DIR, "case.json"), "r") as f: case_sid = json.loads(f.read())["submitter_id"] + assert case_sid for fname in data_fnames: with open(os.path.join(DATA_DIR, fname), "r") as f: resp = client.post(path, headers=submitter, data=f.read()) - assert resp.status_code == 201, resp.data + resp_data = json.loads(resp.data) + # could already exist in the DB. + condition_to_check = (resp.status_code == 201 and resp.data) or ( + resp.status_code == 400 + and "already exists in the DB" + in resp_data["entities"][0]["errors"][0]["message"] + ) + assert condition_to_check, resp.data def post_example_entities_together(client, submitter, data_fnames2=None): @@ -350,9 +358,17 @@ def put_example_entities_together(client, headers): def test_post_example_entities_together(client, pg_driver, cgci_blgsp, submitter): with open(os.path.join(DATA_DIR, "case.json"), "r") as f: case_sid = json.loads(f.read())["submitter_id"] + assert case_sid resp = post_example_entities_together(client, submitter) print(resp.data) - assert resp.status_code == 201, resp.data + resp_data = json.loads(resp.data) + # could already exist in the DB. + condition_to_check = (resp.status_code == 201 and resp.data) or ( + resp.status_code == 400 + and "already exists in the DB" + in resp_data["entities"][0]["errors"][0]["message"] + ) + assert condition_to_check, resp.data def test_dictionary_list_entries(client, pg_driver, cgci_blgsp, submitter): @@ -409,7 +425,10 @@ def test_put_dry_run(client, pg_driver, cgci_blgsp, submitter): assert resp.status_code == 200, resp.data resp_json = json.loads(resp.data) assert resp_json["entity_error_count"] == 0 - assert resp_json["created_entity_count"] == 1 + condition_to_check = ( + resp_json["created_entity_count"] == 1 or resp_json["updated_entity_count"] == 1 + ) + assert condition_to_check with pg_driver.session_scope(): assert not pg_driver.nodes(md.Experiment).first() @@ -455,15 +474,33 @@ def test_insert_multiple_parents_and_export_by_ids( headers = submitter headers["Content-Type"] = "text/tsv" resp = client.post(path, headers=headers, data=f.read()) - assert resp.status_code == 201, resp.data - data = json.loads(resp.data) - submitted_id = data["entities"][0]["id"] + resp_data = json.loads(resp.data) + # could already exist in the DB. + condition_to_check = (resp.status_code == 201 and resp.data) or ( + resp.status_code == 400 + and "already exists in the DB" + in resp_data["entities"][0]["errors"][0]["message"] + ) + assert condition_to_check, resp.data + + # check db for matching experimental metadata + with pg_driver.session_scope(): + filtered = ( + pg_driver.nodes(md.ExperimentalMetadata) + .prop_in( + "submitter_id", + ["BLGSP-71-experimental-01-c", "BLGSP-71-experimental-01-a"], + ) + .all() + ) + submitted_ids = ",".join([node.node_id for node in filtered]) resp = client.get( - "/v0/submission/CGCI/BLGSP/export/?ids={}".format(submitted_id), headers=headers + "/v0/submission/CGCI/BLGSP/export/?ids={}".format(submitted_ids), + headers=headers, ) str_data = str(resp.data) - assert "BLGSP-71-experiment-01" in str_data - assert "BLGSP-71-experiment-02" in str_data + assert "BLGSP-71-experimental-01-a" in str_data + assert "BLGSP-71-experimental-01-c" in str_data assert "experiments.submitter_id" in str_data diff --git a/tests/integration/datadictwithobjid/submission/test_upload.py b/tests/integration/datadictwithobjid/submission/test_upload.py index 4ff4af7e0..450668983 100644 --- a/tests/integration/datadictwithobjid/submission/test_upload.py +++ b/tests/integration/datadictwithobjid/submission/test_upload.py @@ -67,13 +67,17 @@ def submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=None): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_not_indexed( create_alias, create_index, @@ -120,17 +124,21 @@ def test_data_file_not_indexed( data = r.json assert data and len(data) == 1 - assert did == None + assert did is None @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_not_indexed_id_provided( create_alias, create_index, @@ -153,11 +161,13 @@ def test_data_file_not_indexed_id_provided( file = copy.deepcopy(DEFAULT_METADATA_FILE) file["object_id"] = DEFAULT_UUID - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=file + ) # noqa: E501 # index creation assert create_index.call_count == 1 - args, kwargs = create_index.call_args_list[0] + args, kwargs = create_index.call_args_list[0] # pylint: disable=W0612 assert "did" in kwargs did = kwargs["did"] assert "hashes" in kwargs @@ -181,13 +191,17 @@ def test_data_file_not_indexed_id_provided( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_already_indexed( create_alias, create_index, @@ -242,13 +256,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_already_indexed_id_provided( create_alias, create_index, @@ -282,7 +300,9 @@ def get_index_by_uuid(uuid): file = copy.deepcopy(DEFAULT_METADATA_FILE) file["id"] = document.did - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=file + ) # noqa: E501 # no index or alias creation assert not create_index.called @@ -300,13 +320,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url( create_alias, create_index, @@ -365,13 +389,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_multiple_urls( create_alias, create_index, @@ -420,7 +448,8 @@ def get_index_by_uuid(uuid): assert not create_index.called assert not create_alias.called - # make sure original url and new url are in the document and patch gets called + # make sure original url and new url are in the document + # and patch gets called assert DEFAULT_URL in document.urls assert new_url in document.urls assert another_new_url in document.urls @@ -436,13 +465,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_id_provided( create_alias, create_index, @@ -481,6 +514,7 @@ def get_index_by_uuid(uuid): updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file["object_id"] = "14fd1746-61bb-401a-96d2-342cfaf70000" updated_file["urls"] = new_url + resp = submit_metadata_file( client, pg_driver, submitter, cgci_blgsp, data=updated_file ) @@ -500,15 +534,19 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity._update_acl_uploader_for_file" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._update_acl_uploader_for_file" # noqa: E501 ) def test_data_file_already_indexed_object_id_provided_hash_match( update_acl_uploader_indexd, @@ -557,7 +595,9 @@ def get_index_by_uuid(uuid): get_index_uuid.side_effect = get_index_by_uuid - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=file + ) # noqa: E501 # no index or alias creation assert not create_index.called @@ -582,15 +622,19 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity._update_acl_uploader_for_file" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._update_acl_uploader_for_file" # noqa: E501 ) def test_data_file_already_indexed_object_id_provided_hash_match_populated_acl( update_acl_uploader_indexd, @@ -639,7 +683,9 @@ def get_index_by_uuid(uuid): get_index_uuid.side_effect = get_index_by_uuid - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=file + ) # noqa: E501 # no index or alias creation assert not create_index.called @@ -663,17 +709,21 @@ def get_index_by_uuid(uuid): assert not document.authz -""" ----- TESTS THAT SHOULD RESULT IN SUBMISSION FAILURES ARE BELOW ----- """ +# ----- TESTS THAT SHOULD RESULT IN SUBMISSION FAILURES ARE BELOW ----- @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_invalid_id( create_alias, create_index, @@ -727,13 +777,17 @@ def test_data_file_update_url_invalid_id( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_id_provided_different_file_not_indexed( create_alias, create_index, @@ -795,13 +849,17 @@ def test_data_file_update_url_id_provided_different_file_not_indexed( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_different_file_not_indexed( create_alias, create_index, @@ -840,7 +898,7 @@ def test_data_file_update_url_different_file_not_indexed( resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp) - entity = assert_single_entity_from_response(resp) + assert_single_entity_from_response(resp) # now submit again but change url new_url = "some/new/url/location/to/add" @@ -867,13 +925,17 @@ def test_data_file_update_url_different_file_not_indexed( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_update_url_id_provided_different_file_already_indexed( create_alias, create_index, @@ -903,7 +965,9 @@ def test_data_file_update_url_id_provided_different_file_already_indexed( document_with_id.urls = [DEFAULT_URL] different_file_matching_hash_and_size = MagicMock() - different_file_matching_hash_and_size.did = "14fd1746-61bb-401a-96d2-342cfaf70000" + different_file_matching_hash_and_size.did = ( + "14fd1746-61bb-401a-96d2-342cfaf70000" # noqa: E501 + ) different_file_matching_hash_and_size.urls = [DEFAULT_URL] get_index_uuid.return_value = document_with_id @@ -938,13 +1002,17 @@ def test_data_file_update_url_id_provided_different_file_already_indexed( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_create_file_no_required_index( create_alias, create_index, @@ -958,7 +1026,8 @@ def test_create_file_no_required_index( ): """ With REQUIRE_FILE_INDEX_EXISTS = True. - Test submitting a data file that does not exist in indexd (should raise an error and should not create an index or an alias). + Test submitting a data file that does not exist in indexd + (should raise an error and should not create an index or an alias). """ submit_first_experiment(client, pg_driver, submitter, cgci_blgsp) @@ -989,13 +1058,17 @@ def test_create_file_no_required_index( @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_already_indexed_object_id_provided_hash_no_match( create_alias, create_index, @@ -1041,7 +1114,9 @@ def get_index_by_uuid(uuid): get_index_uuid.side_effect = get_index_by_uuid - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=file + ) # noqa: E501 # no index or alias creation assert not create_index.called @@ -1050,6 +1125,7 @@ def get_index_by_uuid(uuid): # response assert_negative_response(resp) entity = assert_single_entity_from_response(resp) + assert entity # check that the acl and uploader fields have NOT been updated in indexd assert not document.acl @@ -1057,13 +1133,17 @@ def get_index_by_uuid(uuid): @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_hash" # noqa: E501 ) @patch( - "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" + "sheepdog.transactions.upload.sub_entities.FileUploadEntity.get_file_from_index_by_uuid" # noqa: E501 ) -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index") -@patch("sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias") +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_index" +) # noqa: E501 +@patch( + "sheepdog.transactions.upload.sub_entities.FileUploadEntity._create_alias" +) # noqa: E501 def test_data_file_already_indexed_object_id_provided_no_hash( create_alias, create_index, @@ -1110,7 +1190,9 @@ def get_index_by_uuid(uuid): get_index_uuid.side_effect = get_index_by_uuid - resp = submit_metadata_file(client, pg_driver, submitter, cgci_blgsp, data=file) + resp = submit_metadata_file( + client, pg_driver, submitter, cgci_blgsp, data=file + ) # noqa: E501 # no index or alias creation assert not create_index.called @@ -1118,7 +1200,7 @@ def get_index_by_uuid(uuid): # response assert_negative_response(resp) - entity = assert_single_entity_from_response(resp) + assert_single_entity_from_response(resp) # check that the acl and uploader fields have NOT been updated in indexd assert not document.acl