diff --git a/noxfile.py b/noxfile.py index b6e34730..a973d2f9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,10 +18,12 @@ from __future__ import absolute_import +from functools import wraps import os import pathlib import re import shutil +import time from typing import Dict, List import warnings @@ -47,6 +49,7 @@ "asyncmock", "pytest", "pytest-cov", + "pytest-xdist", "pytest-asyncio", ] UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] @@ -83,6 +86,7 @@ "mock", "pytest", "google-cloud-testutils", + "pytest-xdist", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] @@ -110,6 +114,26 @@ CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() +def _calculate_duration(func): + """This decorator prints the execution time for the decorated function.""" + + @wraps(func) + def wrapper(*args, **kwargs): + start = time.monotonic() + result = func(*args, **kwargs) + end = time.monotonic() + total_seconds = round(end - start) + hours = total_seconds // 3600 # Integer division to get hours + remaining_seconds = total_seconds % 3600 # Modulo to find remaining seconds + minutes = remaining_seconds // 60 + seconds = remaining_seconds % 60 + human_time = f"{hours:}:{minutes:0>2}:{seconds:0>2}" + print(f"Session ran in {total_seconds} seconds ({human_time})") + return result + + return wrapper + + nox.options.sessions = [ "unit", "system", @@ -128,6 +152,7 @@ @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def lint(session): """Run linters. @@ -144,6 +169,7 @@ def lint(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def blacken(session): """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) @@ -154,6 +180,7 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def format(session): """ Run isort to sort imports. Then run black @@ -174,6 +201,7 @@ def format(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" session.install("docutils", "pygments") @@ -209,6 +237,7 @@ def install_unittest_dependencies(session, *constraints): @nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@_calculate_duration @nox.parametrize( "protobuf_implementation", ["python", "upb", "cpp"], @@ -241,6 +270,7 @@ def unit(session, protobuf_implementation, install_extras=True): # Run py.test against the unit tests. session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=sqlalchemy_bigquery", @@ -288,6 +318,7 @@ def install_systemtest_dependencies(session, *constraints): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@_calculate_duration def system(session): """Run the system test suite.""" constraints_path = str( @@ -315,6 +346,7 @@ def system(session): if system_test_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -323,6 +355,7 @@ def system(session): if system_test_folder_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, @@ -331,6 +364,7 @@ def system(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@_calculate_duration def system_noextras(session): """Run the system test suite.""" constraints_path = str( @@ -360,6 +394,7 @@ def system_noextras(session): if system_test_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -368,6 +403,7 @@ def system_noextras(session): if system_test_folder_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, @@ -376,6 +412,7 @@ def system_noextras(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) +@_calculate_duration def compliance(session): """Run the SQLAlchemy dialect-compliance system tests""" constraints_path = str( @@ -393,6 +430,7 @@ def compliance(session): session.install( "mock", "pytest", + "pytest-xdist", "pytest-rerunfailures", "google-cloud-testutils", "-c", @@ -410,10 +448,11 @@ def compliance(session): session.run( "py.test", + "-n=2", "-vv", f"--junitxml=compliance_{session.python}_sponge_log.xml", - "--reruns=3", - "--reruns-delay=60", + "--reruns=0", + "--reruns-delay=1", "--only-rerun=Exceeded rate limits", "--only-rerun=Already Exists", "--only-rerun=Not found", @@ -430,6 +469,7 @@ def compliance(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def cover(session): """Run the final coverage report. @@ -443,6 +483,7 @@ def cover(session): @nox.session(python="3.10") +@_calculate_duration def docs(session): """Build the docs for this library.""" @@ -480,6 +521,7 @@ def docs(session): @nox.session(python="3.10") +@_calculate_duration def docfx(session): """Build the docfx yaml files for this library.""" @@ -532,6 +574,7 @@ def docfx(session): "protobuf_implementation", ["python", "upb", "cpp"], ) +@_calculate_duration def prerelease_deps(session, protobuf_implementation): """Run all tests with prerelease versions of dependencies installed.""" @@ -616,6 +659,7 @@ def prerelease_deps(session, protobuf_implementation): if os.path.exists(system_test_path): session.run( "py.test", + "-n=auto", "--verbose", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -627,6 +671,7 @@ def prerelease_deps(session, protobuf_implementation): if os.path.exists(system_test_folder_path): session.run( "py.test", + "-n=auto", "--verbose", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index 4008a7e1..18a457c5 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -70,6 +70,25 @@ TABLE_VALUED_ALIAS_ALIASES = "bigquery_table_valued_alias_aliases" +from sqlalchemy.testing.provision import create_db, drop_db +@create_db.for_db("bigquery") +def _bigquery_create_db(cfg, eng, ident): + # This and _bigquery_drop_db are adequate to get past the errors generated when using xdist: + # NotImplementedError: no DB creation routine for cfg + # NotImplementedError: no DB drop routine for cfg + # + # HOWEVER... we need to populate this in some way to create dbs that + # work. THere are examples of create_db style functions in various + # dialects provision.py files. + # This sqlalchemy issue speaks to this problem: + # https://github.com/sqlalchemy/sqlalchemy/discussions/10948 + pass + +@drop_db.for_db("bigquery") +def _bigquery_drop_db(cfg, eng, ident): + pass # _drop_dbs_w_ident(eng.url.database, eng.driver, ident) + # SEE note above for create_db + def assert_(cond, message="Assertion failed"): # pragma: NO COVER if not cond: