diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index f0f3b24b..98994f47 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f946c75373c2b0040e8e318c5e85d0cf46bc6e61d0a01f3ef94d8de974ac6790 + digest: sha256:2d816f26f728ac8b24248741e7d4c461c09764ef9f7be3684d557c9632e46dbd +# created: 2023-06-28T17:03:33.371210701Z diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 5624d4e6..0982525d 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -38,3 +38,12 @@ env_vars: { key: "SECRET_MANAGER_KEYS" value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } + +# Store the packages we uploaded to PyPI. That way, we have a record of exactly +# what we published, which we can use to generate SBOMs and attestations. +action { + define_artifacts { + regex: "github/python-bigquery-sqlalchemy/**/*.tar.gz" + strip_prefix: "github/python-bigquery-sqlalchemy" + } +} diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in index cbd7e77f..ec867d9f 100644 --- a/.kokoro/requirements.in +++ b/.kokoro/requirements.in @@ -1,10 +1,10 @@ gcp-docuploader -gcp-releasetool +gcp-releasetool>=1.10.5 # required for compatibility with cryptography>=39.x importlib-metadata typing-extensions twine wheel setuptools -nox +nox>=2022.11.21 # required to remove dependency on py charset-normalizer<3 click<8.1.0 diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 05dc4672..c7929db6 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements.in # @@ -113,33 +113,26 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==38.0.3 \ - --hash=sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d \ - --hash=sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd \ - --hash=sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146 \ - --hash=sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7 \ - --hash=sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436 \ - --hash=sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0 \ - --hash=sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828 \ - --hash=sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b \ - --hash=sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55 \ - --hash=sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36 \ - --hash=sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50 \ - --hash=sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2 \ - --hash=sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a \ - --hash=sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8 \ - --hash=sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0 \ - --hash=sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548 \ - --hash=sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320 \ - --hash=sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748 \ - --hash=sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249 \ - --hash=sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959 \ - --hash=sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f \ - --hash=sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0 \ - --hash=sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd \ - --hash=sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220 \ - --hash=sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c \ - --hash=sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722 +cryptography==41.0.0 \ + --hash=sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55 \ + --hash=sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895 \ + --hash=sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be \ + --hash=sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928 \ + --hash=sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d \ + --hash=sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8 \ + --hash=sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237 \ + --hash=sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9 \ + --hash=sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78 \ + --hash=sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d \ + --hash=sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0 \ + --hash=sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46 \ + --hash=sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5 \ + --hash=sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4 \ + --hash=sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d \ + --hash=sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75 \ + --hash=sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb \ + --hash=sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2 \ + --hash=sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be # via # gcp-releasetool # secretstorage @@ -159,9 +152,9 @@ gcp-docuploader==0.6.4 \ --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \ --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf # via -r requirements.in -gcp-releasetool==1.10.0 \ - --hash=sha256:72a38ca91b59c24f7e699e9227c90cbe4dd71b789383cb0164b088abae294c83 \ - --hash=sha256:8c7c99320208383d4bb2b808c6880eb7a81424afe7cdba3c8d84b25f4f0e097d +gcp-releasetool==1.10.5 \ + --hash=sha256:174b7b102d704b254f2a26a3eda2c684fd3543320ec239baf771542a2e58e109 \ + --hash=sha256:e29d29927fe2ca493105a82958c6873bb2b90d503acac56be2c229e74de0eec9 # via -r requirements.in google-api-core==2.10.2 \ --hash=sha256:10c06f7739fe57781f87523375e8e1a3a4674bf6392cd6131a3222182b971320 \ @@ -340,9 +333,9 @@ more-itertools==9.0.0 \ --hash=sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41 \ --hash=sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab # via jaraco-classes -nox==2022.8.7 \ - --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ - --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c +nox==2022.11.21 \ + --hash=sha256:0e41a990e290e274cb205a976c4c97ee3c5234441a8132c8c3fd9ea3c22149eb \ + --hash=sha256:e21c31de0711d1274ca585a2c5fde36b1aa962005ba8e9322bf5eeed16dcd684 # via -r requirements.in packaging==21.3 \ --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ @@ -385,10 +378,6 @@ protobuf==3.20.3 \ # gcp-docuploader # gcp-releasetool # google-api-core -py==1.11.0 \ - --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ - --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 - # via nox pyasn1==0.4.8 \ --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba @@ -428,9 +417,9 @@ readme-renderer==37.3 \ --hash=sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273 \ --hash=sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343 # via twine -requests==2.28.1 \ - --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ - --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via # gcp-releasetool # google-api-core diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a280f8a..42a12b52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,27 @@ Older versions of this project were distributed as [pybigquery][0]. [2]: https://pypi.org/project/pybigquery/#history +## [1.7.0](https://github.com/googleapis/python-bigquery-sqlalchemy/compare/v1.6.1...v1.7.0) (2023-07-11) + + +### Features + +* Added regexp_match operator support for bigquery ([#511](https://github.com/googleapis/python-bigquery-sqlalchemy/issues/511)) ([fd78093](https://github.com/googleapis/python-bigquery-sqlalchemy/commit/fd780938ca4a0d32448ed666753360f05584d2ab)) +* Remove pyarrow and bqstorage as dependencies ([#847](https://github.com/googleapis/python-bigquery-sqlalchemy/issues/847)) ([5d6b38c](https://github.com/googleapis/python-bigquery-sqlalchemy/commit/5d6b38c94dfef0bd0dbf5037ad21a352d2bd8e4f)) + + +### Bug Fixes + +* Avoid aliasing known CTEs ([#839](https://github.com/googleapis/python-bigquery-sqlalchemy/issues/839)) ([8a1f694](https://github.com/googleapis/python-bigquery-sqlalchemy/commit/8a1f694e59f5bac5ce66d9f8a04066980d7f9893)) +* Ensure correct alter table alter column statement is generated on data type changes in alembic ([#845](https://github.com/googleapis/python-bigquery-sqlalchemy/issues/845)) ([493430a](https://github.com/googleapis/python-bigquery-sqlalchemy/commit/493430a58914dc89fa5c16ea2127e4bed58bd0bf)) +* Remove "future" dependency ([#542](https://github.com/googleapis/python-bigquery-sqlalchemy/issues/542)) ([ba5e244](https://github.com/googleapis/python-bigquery-sqlalchemy/commit/ba5e24487e798fd77e7db93321c374a9196293f4)) +* Remove type annotations from _struct.py ([#733](https://github.com/googleapis/python-bigquery-sqlalchemy/issues/733)) ([27814df](https://github.com/googleapis/python-bigquery-sqlalchemy/commit/27814dfbea8267d1b7abbc47fbbb56bb36f3585d)) + + +### Documentation + +* Pass credentials as python dictionary ([#737](https://github.com/googleapis/python-bigquery-sqlalchemy/issues/737)) ([074321d](https://github.com/googleapis/python-bigquery-sqlalchemy/commit/074321ddaa10001773e7e6044f4a0df1bb530331)) + ## [1.6.1](https://github.com/googleapis/python-bigquery-sqlalchemy/compare/v1.6.0...v1.6.1) (2023-02-01) diff --git a/README.rst b/README.rst index d4b4ea68..0e0d30bc 100644 --- a/README.rst +++ b/README.rst @@ -81,6 +81,20 @@ Windows \Scripts\activate \Scripts\pip.exe install sqlalchemy-bigquery + +Installations when processing large datasets +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When handling large datasets, you may see speed increases by also installing the +`bqstorage` dependencies. See the instructions above about creating a virtual +environment and then install `sqlalchemy-bigquery` using the `bqstorage` extras: + +.. code-block:: console + + source /bin/activate + /bin/pip install sqlalchemy-bigquery[bqstorage] + + Usage ----- @@ -104,12 +118,27 @@ Project Authentication ^^^^^^^^^^^^^^ -Follow the `Google Cloud library guide `_ for authentication. Alternatively, you can provide the path to a service account JSON file in ``create_engine()``: +Follow the `Google Cloud library guide `_ for authentication. + +Alternatively, you can choose either of the following approaches: + +* provide the path to a service account JSON file in ``create_engine()`` using the ``credentials_path`` parameter: .. code-block:: python + # provide the path to a service account JSON file engine = create_engine('bigquery://', credentials_path='/path/to/keyfile.json') +* pass the credentials in ``create_engine()`` as a Python dictionary using the ``credentials_info`` parameter: + +.. code-block:: python + + # provide credentials as a Python dictionary + credentials_info = { + "type": "service_account", + "project_id": "your-service-account-project-id" + }, + engine = create_engine('bigquery://', credentials_info=credentials_info) Location ^^^^^^^^ diff --git a/dev_requirements.txt b/dev_requirements.txt index 17e4e6d0..ddc53054 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,5 @@ -sqlalchemy>=1.1.9,<2.0.0 +sqlalchemy>=2.0.15,<2.1.0 google-cloud-bigquery>=1.6.0 -future==0.18.2 pytest===6.2.5 pytest-flake8===1.1.0 # versions 1.1.1 and above require pytest 7 -pytz==2022.2.1 +pytz==2023.3 diff --git a/docs/alembic.rst b/docs/alembic.rst index 8b5df741..70925b1e 100644 --- a/docs/alembic.rst +++ b/docs/alembic.rst @@ -15,7 +15,7 @@ Supported operations: `add_column(table_name, column, schema=None) `_ -`alter_column(table_name, column_name, nullable=None, schema=None) +`alter_column(table_name, column_name, nullable=None, schema=None, type_=None) `_ `bulk_insert(table, rows, multiinsert=True) diff --git a/noxfile.py b/noxfile.py index 2c4cc035..ee728ffe 100644 --- a/noxfile.py +++ b/noxfile.py @@ -50,10 +50,12 @@ "3.8": [ "tests", "alembic", + "bqstorage", ], "3.11": [ "tests", "geography", + "bqstorage", ], } @@ -73,10 +75,12 @@ "3.8": [ "tests", "alembic", + "bqstorage", ], "3.11": [ "tests", "geography", + "bqstorage", ], } @@ -179,7 +183,7 @@ def install_unittest_dependencies(session, *constraints): session.install("-e", ".", *constraints) -def default(session): +def default(session, install_extras=True): # Install all test dependencies, then install this package in-place. constraints_path = str( @@ -187,6 +191,14 @@ def default(session): ) install_unittest_dependencies(session, "-c", constraints_path) + if install_extras and session.python == "3.11": + install_target = ".[geography,alembic,tests,bqstorage]" + elif install_extras: + install_target = ".[all]" + else: + install_target = "." + session.install("-e", install_target, "-c", constraints_path) + # Run py.test against the unit tests. session.run( "py.test", @@ -283,6 +295,51 @@ def system(session): ) +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system_noextras(session): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + global SYSTEM_TEST_EXTRAS_BY_PYTHON + SYSTEM_TEST_EXTRAS_BY_PYTHON = False + install_systemtest_dependencies(session, "-c", constraints_path) + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if system_test_folder_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) + + @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) def compliance(session): """Run the SQLAlchemy dialect-compliance system tests""" @@ -302,9 +359,7 @@ def compliance(session): session.install("--pre", "--no-deps", "--upgrade", "sqlalchemy<2.0.0") session.install( "mock", - # TODO: Allow latest version of pytest once SQLAlchemy 1.4.28+ is supported. - # See: https://github.com/googleapis/python-bigquery-sqlalchemy/issues/413 - "pytest<=7.0.0dev", + "pytest", "pytest-rerunfailures", "google-cloud-testutils", "-c", @@ -318,6 +373,8 @@ def compliance(session): extras = "[tests]" session.install("-e", f".{extras}", "-c", constraints_path) + session.run("python", "-m", "pip", "freeze") + session.run( "py.test", "-vv", @@ -385,12 +442,11 @@ def docfx(session): session.install("-e", ".") session.install( - "sphinx==4.0.1", + "gcp-sphinx-docfx-yaml", "alabaster", "geoalchemy2", "shapely", "recommonmark", - "gcp-sphinx-docfx-yaml", ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) @@ -463,6 +519,7 @@ def prerelease_deps(session): "grpcio!=1.52.0rc1", "grpcio-status", "google-api-core", + "google-auth", "proto-plus", "google-cloud-testutils", # dependencies of google-cloud-testutils" @@ -475,7 +532,6 @@ def prerelease_deps(session): # Remaining dependencies other_deps = [ "requests", - "google-auth", ] session.install(*other_deps) diff --git a/owlbot.py b/owlbot.py index 1ea90868..4cc564d4 100644 --- a/owlbot.py +++ b/owlbot.py @@ -29,8 +29,8 @@ # ---------------------------------------------------------------------------- extras = ["tests"] extras_by_python = { - "3.8": ["tests", "alembic"], - "3.11": ["tests", "geography"], + "3.8": ["tests", "alembic", "bqstorage"], + "3.11": ["tests", "geography", "bqstorage"], } templated_files = common.py_library( unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11"], @@ -95,6 +95,14 @@ ) +s.replace( + ["noxfile.py"], + r"def default\(session\)", + "def default(session, install_extras=True)", +) + + + def place_before(path, text, *before_text, escape=None): replacement = "\n".join(before_text) + "\n" + text @@ -117,6 +125,23 @@ def place_before(path, text, *before_text, escape=None): ) +install_logic = ''' + if install_extras and session.python == "3.11": + install_target = ".[geography,alembic,tests,bqstorage]" + elif install_extras: + install_target = ".[all]" + else: + install_target = "." + session.install("-e", install_target, "-c", constraints_path) +''' + +place_before( + "noxfile.py", + "# Run py.test against the unit tests.", + install_logic, +) + + # Maybe we can get rid of this when we don't need pytest-rerunfailures, # which we won't need when BQ retries itself: # https://github.com/googleapis/python-bigquery/pull/837 @@ -140,9 +165,7 @@ def compliance(session): session.install("--pre", "--no-deps", "--upgrade", "sqlalchemy<2.0.0") session.install( "mock", - # TODO: Allow latest version of pytest once SQLAlchemy 1.4.28+ is supported. - # See: https://github.com/googleapis/python-bigquery-sqlalchemy/issues/413 - "pytest<=7.0.0dev", + "pytest", "pytest-rerunfailures", "google-cloud-testutils", "-c", @@ -156,6 +179,8 @@ def compliance(session): extras = "[tests]" session.install("-e", f".{extras}", "-c", constraints_path) + session.run("python", "-m", "pip", "freeze") + session.run( "py.test", "-vv", @@ -189,6 +214,63 @@ def compliance(session): s.replace(["noxfile.py"], '"alabaster"', '"alabaster", "geoalchemy2", "shapely"') +system_noextras = ''' +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system_noextras(session): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + global SYSTEM_TEST_EXTRAS_BY_PYTHON + SYSTEM_TEST_EXTRAS_BY_PYTHON = False + install_systemtest_dependencies(session, "-c", constraints_path) + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if system_test_folder_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) + + +''' + + +place_before( + "noxfile.py", + "@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1])\n" + "def compliance(session):", + system_noextras, + escape="()[]", + ) + # Add DB config for SQLAlchemy dialect test suite. # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/89 diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index bd43d0d7..4e326e01 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,16 +1,16 @@ -attrs==22.1.0 -click==8.1.3 -google-auth==2.11.0 +attrs==23.1.0 +click==8.1.4 +google-auth==2.21.0 google-cloud-testutils==1.3.3 -iniconfig==1.1.1 -packaging==21.3 -pluggy==1.0.0 +iniconfig==2.0.0 +packaging==23.1 +pluggy==1.2.0 py==1.11.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pyparsing==3.0.9 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pyparsing==3.1.0 pytest===6.2.5 rsa==4.9 six==1.16.0 toml==0.10.2 -typing-extensions==4.3.0 +typing-extensions==4.7.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 48a238da..e4e437c5 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,34 +1,34 @@ -alembic==1.8.1 -certifi==2022.6.15 -charset-normalizer==2.1.1 -future==0.18.2 -geoalchemy2==0.12.5 -google-api-core[grpc]==2.10.0 -google-auth==2.11.0 -google-cloud-bigquery==3.3.2 -google-cloud-core==2.3.2 +alembic==1.11.1 +certifi==2023.5.7 +charset-normalizer==3.2.0 +geoalchemy2==0.14.0 +google-api-core[grpc]==2.11.1 +google-auth==2.21.0 +google-cloud-bigquery==3.11.3 +google-cloud-core==2.3.3 google-crc32c==1.5.0 -google-resumable-media==2.3.3 -googleapis-common-protos==1.56.4 -greenlet==1.1.3 -grpcio==1.48.1 -grpcio-status==1.48.1 -idna==3.3 -importlib-resources==5.9.0 -mako==1.2.2 -markupsafe==2.1.1 -packaging==21.3 -proto-plus==1.22.1 -protobuf==4.21.5 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pyparsing==3.0.9 +google-resumable-media==2.5.0 +googleapis-common-protos==1.59.1 +greenlet==2.0.2 +grpcio==1.56.0 +grpcio-status==1.56.0 +idna==3.4 +importlib-resources===5.12.0; python_version == '3.7' +importlib-resources==6.0.0; python_version >= '3.8' +mako==1.2.4 +markupsafe==2.1.3 +packaging==23.1 +proto-plus==1.22.3 +protobuf==4.23.4 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pyparsing==3.1.0 python-dateutil==2.8.2 -pytz==2022.2.1 -requests==2.28.1 +pytz==2023.3 +requests==2.31.0 rsa==4.9 -shapely==1.8.4 +shapely==2.0.1 six==1.16.0 sqlalchemy===1.4.27 -typing-extensions==4.3.0 +typing-extensions==4.7.1 urllib3==1.26.12 diff --git a/setup.py b/setup.py index bed5195c..0ed6037d 100644 --- a/setup.py +++ b/setup.py @@ -45,11 +45,26 @@ def readme(): return f.read() -extras = dict( - geography=["GeoAlchemy2", "shapely"], - alembic=["alembic"], - tests=["packaging", "pytz"], -) +extras = { + "geography": ["GeoAlchemy2", "shapely"], + "alembic": ["alembic"], + "tests": ["packaging", "pytz"], + # Keep the no-op bqstorage extra for backward compatibility. + # See: https://github.com/googleapis/python-bigquery/issues/757 + "bqstorage": [ + "google-cloud-bigquery-storage >= 2.0.0, <3.0.0dev", + # Due to an issue in pip's dependency resolver, the `grpc` extra is not + # installed, even though `google-cloud-bigquery-storage` specifies it + # as `google-api-core[grpc]`. We thus need to explicitly specify it here. + # See: https://github.com/googleapis/python-bigquery/issues/83 The + # grpc.Channel.close() method isn't added until 1.32.0. + # https://github.com/grpc/grpc/pull/15254 + "grpcio >= 1.47.0, < 2.0dev", + "grpcio >= 1.49.1, < 2.0dev; python_version>='3.11'", + "pyarrow >= 3.0.0", + ], +} + extras["all"] = set(itertools.chain.from_iterable(extras.values())) setup( @@ -85,11 +100,8 @@ def readme(): # https://github.com/googleapis/google-cloud-python/issues/10566 "google-auth>=1.25.0,<3.0.0dev", # Work around pip wack. "google-cloud-bigquery>=2.25.2,<4.0.0dev", - "google-cloud-bigquery-storage>=2.0.0,<3.0.0dev", "packaging", - "pyarrow>=3.0.0", "sqlalchemy>=1.2.0,<2.0.0dev", - "future", ], extras_require=extras, python_requires=">=3.7, <3.12", diff --git a/sqlalchemy_bigquery/__init__.py b/sqlalchemy_bigquery/__init__.py index 2739dfa2..55253049 100644 --- a/sqlalchemy_bigquery/__init__.py +++ b/sqlalchemy_bigquery/__init__.py @@ -68,7 +68,7 @@ try: from .geography import GEOGRAPHY, WKB, WKT # noqa -except ImportError: +except ImportError: # pragma: NO COVER pass else: __all__.extend(["GEOGRAPHY", "WKB", "WKT"]) diff --git a/sqlalchemy_bigquery/_struct.py b/sqlalchemy_bigquery/_struct.py index 6ebb5a64..fc551c12 100644 --- a/sqlalchemy_bigquery/_struct.py +++ b/sqlalchemy_bigquery/_struct.py @@ -17,8 +17,6 @@ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from typing import Mapping, Tuple - import packaging.version import sqlalchemy.sql.default_comparator import sqlalchemy.sql.sqltypes @@ -54,8 +52,8 @@ class STRUCT(sqlalchemy.sql.sqltypes.Indexable, sqlalchemy.types.UserDefinedType def __init__( self, - *fields: Tuple[str, sqlalchemy.types.TypeEngine], - **kwfields: Mapping[str, sqlalchemy.types.TypeEngine], + *fields, + **kwfields, ): # Note that because: # https://docs.python.org/3/whatsnew/3.6.html#pep-468-preserving-keyword-argument-order diff --git a/sqlalchemy_bigquery/_types.py b/sqlalchemy_bigquery/_types.py index 4e18dc2a..8399e978 100644 --- a/sqlalchemy_bigquery/_types.py +++ b/sqlalchemy_bigquery/_types.py @@ -24,7 +24,7 @@ try: from .geography import GEOGRAPHY -except ImportError: +except ImportError: # pragma: NO COVER pass from ._struct import STRUCT @@ -69,7 +69,7 @@ try: _type_map["GEOGRAPHY"] = GEOGRAPHY -except NameError: +except NameError: # pragma: NO COVER pass STRUCT_FIELD_TYPES = "RECORD", "STRUCT" diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index 843f6312..a3496f93 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -19,9 +19,6 @@ """Integration between SQLAlchemy and BigQuery.""" -from __future__ import absolute_import -from __future__ import unicode_literals - from decimal import Decimal import random import operator @@ -40,6 +37,7 @@ import sqlalchemy.sql.type_api from sqlalchemy.exc import NoSuchTableError from sqlalchemy import util +from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.compiler import ( SQLCompiler, GenericTypeCompiler, @@ -263,6 +261,7 @@ def _known_tables(self): if isinstance(from_, Table): known_tables.add(from_.name) elif isinstance(from_, CTE): + known_tables.add(from_.name) for column in from_.original.selected_columns: table = getattr(column, "table", None) if table is not None: @@ -553,6 +552,18 @@ def visit_getitem_binary(self, binary, operator_, **kw): right = self.process(binary.right, **kw) return f"{left}[OFFSET({right})]" + def _get_regexp_args(self, binary, kw): + string = self.process(binary.left, **kw) + pattern = self.process(binary.right, **kw) + return string, pattern + + def visit_regexp_match_op_binary(self, binary, operator, **kw): + string, pattern = self._get_regexp_args(binary, kw) + return "REGEXP_CONTAINS(%s, %s)" % (string, pattern) + + def visit_not_regexp_match_op_binary(self, binary, operator, **kw): + return "NOT %s" % self.visit_regexp_match_op_binary(binary, operator, **kw) + class BigQueryTypeCompiler(GenericTypeCompiler): def visit_INTEGER(self, type_, **kw): @@ -774,6 +785,7 @@ def __init__( self.credentials_info = credentials_info self.credentials_base64 = credentials_base64 self.location = location + self.identifier_preparer = self.preparer(self) self.dataset_id = None self.list_tables_page_size = list_tables_page_size @@ -1055,10 +1067,23 @@ def __init__(self, *args, **kwargs): try: import alembic # noqa -except ImportError: +except ImportError: # pragma: NO COVER pass else: from alembic.ddl import impl + from alembic.ddl.base import ColumnType, format_type, alter_table, alter_column class SqlalchemyBigqueryImpl(impl.DefaultImpl): __dialect__ = "bigquery" + + @compiles(ColumnType, "bigquery") + def visit_column_type(element: ColumnType, compiler: DDLCompiler, **kw) -> str: + """Replaces the visit_column_type() function in alembic/alembic/ddl/base.py. + The alembic version ends in TYPE , but bigquery requires this syntax: + SET DATA TYPE """ + + return "%s %s %s" % ( # pragma: NO COVER + alter_table(compiler, element.table_name, element.schema), + alter_column(compiler, element.column_name), + "SET DATA TYPE %s" % format_type(compiler, element.type_), + ) diff --git a/sqlalchemy_bigquery/version.py b/sqlalchemy_bigquery/version.py index b47d6225..5b3e9059 100644 --- a/sqlalchemy_bigquery/version.py +++ b/sqlalchemy_bigquery/version.py @@ -17,4 +17,4 @@ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -__version__ = "1.6.1" +__version__ = "1.7.0" diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt deleted file mode 100644 index 9d2df4fe..00000000 --- a/testing/constraints-3.6.txt +++ /dev/null @@ -1,12 +0,0 @@ -# This constraints file is used to check that lower bounds -# are correct in setup.py -# List *all* library dependencies and extras in this file. -# Pin the version to the lower bound. -# -# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", -sqlalchemy==1.2.0 -google-auth==1.25.0 -google-cloud-bigquery==2.25.2 -google-cloud-bigquery-storage==2.0.0 -google-api-core==1.31.5 -pyarrow==3.0.0 diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 9d2df4fe..1d0a1b72 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -6,7 +6,7 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", sqlalchemy==1.2.0 google-auth==1.25.0 -google-cloud-bigquery==2.25.2 +google-cloud-bigquery==3.3.6 google-cloud-bigquery-storage==2.0.0 google-api-core==1.31.5 pyarrow==3.0.0 diff --git a/tests/system/test_alembic.py b/tests/system/test_alembic.py index 6c4736c8..1948a19a 100644 --- a/tests/system/test_alembic.py +++ b/tests/system/test_alembic.py @@ -20,7 +20,7 @@ import contextlib import pytest -from sqlalchemy import Column, DateTime, Integer, String +from sqlalchemy import Column, DateTime, Integer, String, Numeric import google.api_core.exceptions from google.cloud.bigquery import SchemaField @@ -149,9 +149,7 @@ def test_alembic_scenario(alembic_table): """ ) - # The only thing we can alter about a column is we can make it - # nullable: - op.alter_column("transactions", "amount", True) + op.alter_column("transactions", "amount", nullable=True) assert alembic_table("transactions", "schema") == [ SchemaField("account", "INTEGER", "REQUIRED"), SchemaField("transaction_time", "DATETIME", "REQUIRED"), @@ -162,3 +160,12 @@ def test_alembic_scenario(alembic_table): assert alembic_table("transactions").description == "Transaction log" op.drop_table("transactions") + + # Another thing we can do is alter the datatype of a nullable column, + # if allowed by BigQuery's type coercion rules + op.create_table("identifiers", Column("id", Integer)) + + op.alter_column("identifiers", "id", type_=Numeric) + assert alembic_table("identifiers", "schema") == [SchemaField("id", "NUMERIC")] + + op.drop_table("identifiers") diff --git a/tests/system/test_sqlalchemy_bigquery.py b/tests/system/test_sqlalchemy_bigquery.py index 0772e10a..01078596 100644 --- a/tests/system/test_sqlalchemy_bigquery.py +++ b/tests/system/test_sqlalchemy_bigquery.py @@ -25,7 +25,7 @@ from sqlalchemy.engine import create_engine from sqlalchemy.schema import Table, MetaData, Column from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import types, func, case, inspect +from sqlalchemy import types, func, case, inspect, not_ from sqlalchemy.sql import expression, select, literal_column from sqlalchemy.exc import NoSuchTableError from sqlalchemy.orm import sessionmaker @@ -774,3 +774,81 @@ def test_unnest(engine, bigquery_dataset): f" unnest(`{bigquery_dataset}.test_unnest_1`.`objects`) AS `foo_objects`" ) assert sorted(r[0] for r in conn.execute(query)) == ["a", "b", "c", "x", "y"] + + +@pytest.mark.skipif( + packaging.version.parse(sqlalchemy.__version__) < packaging.version.parse("1.4"), + reason="unnest (and other table-valued-function) support required version 1.4", +) +def test_unnest_with_cte(engine, bigquery_dataset): + from sqlalchemy import select, func, String + from sqlalchemy_bigquery import ARRAY + + conn = engine.connect() + metadata = MetaData() + table_name = "test_unnest_with_cte" + table = Table( + f"{bigquery_dataset}.{table_name}", + metadata, + Column("foo", String), + Column("bars", ARRAY(String)), + ) + metadata.create_all(engine) + conn.execute( + table.insert(), + [dict(foo="first", bars=["a", "b", "c"]), dict(foo="second", bars=["x", "y"])], + ) + selectable = select(table.c).select_from(table).cte("cte") + query = select( + [ + selectable.c.foo, + func.unnest(selectable.c.bars).column_valued("unnest_bars"), + ] + ).select_from(selectable) + compiled = str(query.compile(engine)) + assert " ".join(compiled.strip().split()) == ( + f"WITH `cte` " + f"AS (SELECT `{bigquery_dataset}.{table_name}`.`foo` AS `foo`," + f" `{bigquery_dataset}.{table_name}`.`bars` AS `bars`" + f" FROM `{bigquery_dataset}.{table_name}`) " + f"SELECT `cte`.`foo`, `unnest_bars` " + f"FROM `cte`, unnest(`cte`.`bars`) AS `unnest_bars`" + ) + + assert sorted(r for r in conn.execute(query)) == [ + ("first", "a"), + ("first", "b"), + ("first", "c"), + ("second", "x"), + ("second", "y"), + ] + + +@pytest.mark.skipif( + packaging.version.parse(sqlalchemy.__version__) < packaging.version.parse("1.4"), + reason="regexp_match support requires version 1.4 or higher", +) +def test_regexp_match(session, table): + results = ( + session.query(table.c.string) + .where(table.c.string.regexp_match(".*52 St &.*")) + .all() + ) + + number_of_52st_records = 12 + assert len(results) == number_of_52st_records + + +@pytest.mark.skipif( + packaging.version.parse(sqlalchemy.__version__) < packaging.version.parse("1.4"), + reason="regexp_match support requires version 1.4 or higher", +) +def test_not_regexp_match(session, table): + results = ( + session.query(table.c.string) + .where(not_(table.c.string.regexp_match("^Barrow St & Hudson St$"))) + .all() + ) + + number_of_non_barrowst_records = 993 + assert len(results) == number_of_non_barrowst_records diff --git a/tests/unit/test_geography.py b/tests/unit/test_geography.py index 25d3c605..6924ade0 100644 --- a/tests/unit/test_geography.py +++ b/tests/unit/test_geography.py @@ -104,7 +104,7 @@ def test_geoalchemy2_core(faux_conn, last_query): try: conn.execute( - select([lake_table.c.name, lake_table.c.geog.ST_AREA().label("area")]) + select([lake_table.c.name, lake_table.c.geog.ST_Area().label("area")]) ) except Exception: pass # sqlite had no special functions :) @@ -160,16 +160,18 @@ def test_GEOGRAPHY_ElementType_bad_extended(): def test_GEOGRAPHY_ElementType(): from sqlalchemy_bigquery import GEOGRAPHY, WKB - data = GEOGRAPHY.ElementType("data") + # The data argument here should be composed of hex characters: + # 1-0 and a-f + data = GEOGRAPHY.ElementType("123def") assert isinstance(data, WKB) - assert (data.data, data.srid, data.extended) == ("data", 4326, True) + assert (data.data, data.srid, data.extended) == ("123def", 4326, True) def test_calling_st_functions_that_dont_take_geographies(faux_conn, last_query): from sqlalchemy import select, func try: - faux_conn.execute(select([func.ST_GEOGFROMTEXT("point(0 0)")])) + faux_conn.execute(select([func.ST_GeogFromText("point(0 0)")])) except Exception: pass # sqlite had no special functions :) diff --git a/tests/unit/test_select.py b/tests/unit/test_select.py index 08fc0225..ee5e01cb 100644 --- a/tests/unit/test_select.py +++ b/tests/unit/test_select.py @@ -23,6 +23,7 @@ import packaging.version import pytest import sqlalchemy +from sqlalchemy import not_ import sqlalchemy_bigquery @@ -445,3 +446,43 @@ def test_array_indexing(faux_conn, metadata): ) got = str(sqlalchemy.select([t.c.a[0]]).compile(faux_conn.engine)) assert got == "SELECT `t`.`a`[OFFSET(%(a_1:INT64)s)] AS `anon_1` \nFROM `t`" + + +@pytest.mark.skipif( + packaging.version.parse(sqlalchemy.__version__) < packaging.version.parse("1.4"), + reason="regexp_match support requires version 1.4 or higher", +) +def test_visit_regexp_match_op_binary(faux_conn): + table = setup_table( + faux_conn, + "table", + sqlalchemy.Column("foo", sqlalchemy.String), + ) + + # NOTE: "sample_pattern" is not used in this test, we are not testing + # the regex engine, we are testing the ability to create SQL + sql_statement = table.c.foo.regexp_match("sample_pattern") + result = sql_statement.compile(faux_conn).string + expected = "REGEXP_CONTAINS(`table`.`foo`, %(foo_1:STRING)s)" + + assert result == expected + + +@pytest.mark.skipif( + packaging.version.parse(sqlalchemy.__version__) < packaging.version.parse("1.4"), + reason="regexp_match support requires version 1.4 or higher", +) +def test_visit_not_regexp_match_op_binary(faux_conn): + table = setup_table( + faux_conn, + "table", + sqlalchemy.Column("foo", sqlalchemy.String), + ) + + # NOTE: "sample_pattern" is not used in this test, we are not testing + # the regex engine, we are testing the ability to create SQL + sql_statement = not_(table.c.foo.regexp_match("sample_pattern")) + result = sql_statement.compile(faux_conn).string + expected = "NOT REGEXP_CONTAINS(`table`.`foo`, %(foo_1:STRING)s)" + + assert result == expected