From cb166cc29a9e1774d70597711ed5962f344e3c96 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 5 Jul 2022 18:35:53 +0000 Subject: [PATCH 1/6] chore(python): drop python 3.6 Source-Link: https://github.com/googleapis/synthtool/commit/4f89b13af10d086458f9b379e56a614f9d6dab7b Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c --- .github/.OwlBot.lock.yaml | 4 +- .kokoro/continuous/prerelease-deps.cfg | 7 + .kokoro/presubmit/prerelease-deps.cfg | 7 + .kokoro/test-samples-impl.sh | 4 +- README.rst | 235 +++--------------- noxfile.py | 88 +++++++ samples/snippets/noxfile.py | 2 +- .../templates/install_deps.tmpl.rst | 2 +- 8 files changed, 146 insertions(+), 203 deletions(-) create mode 100644 .kokoro/continuous/prerelease-deps.cfg create mode 100644 .kokoro/presubmit/prerelease-deps.cfg diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 757c9dca..1ce60852 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 -# created: 2022-05-05T22:08:23.383410683Z + digest: sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c +# created: 2022-07-05T18:31:20.838186805Z diff --git a/.kokoro/continuous/prerelease-deps.cfg b/.kokoro/continuous/prerelease-deps.cfg new file mode 100644 index 00000000..3595fb43 --- /dev/null +++ b/.kokoro/continuous/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/presubmit/prerelease-deps.cfg b/.kokoro/presubmit/prerelease-deps.cfg new file mode 100644 index 00000000..3595fb43 --- /dev/null +++ b/.kokoro/presubmit/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 8a324c9c..2c6500ca 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.6 -m pip install --upgrade --quiet nox +python3.9 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" + python3.9 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. diff --git a/README.rst b/README.rst index c3586a4b..5675a6c9 100644 --- a/README.rst +++ b/README.rst @@ -1,23 +1,22 @@ -SQLAlchemy Dialect for BigQuery -=============================== +Python Client for SQLAlchemy dialect for BigQuery API +===================================================== -|GA| |pypi| |versions| +|preview| |pypi| |versions| -`SQLALchemy Dialects`_ +`SQLAlchemy dialect for BigQuery API`_: -- `Dialect Documentation`_ +- `Client Library Documentation`_ - `Product Documentation`_ -.. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg - :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability +.. |preview| image:: https://img.shields.io/badge/support-preview-orange.svg + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels .. |pypi| image:: https://img.shields.io/pypi/v/sqlalchemy-bigquery.svg :target: https://pypi.org/project/sqlalchemy-bigquery/ .. |versions| image:: https://img.shields.io/pypi/pyversions/sqlalchemy-bigquery.svg :target: https://pypi.org/project/sqlalchemy-bigquery/ -.. _SQLAlchemy Dialects: https://docs.sqlalchemy.org/en/14/dialects/ -.. _Dialect Documentation: https://googleapis.dev/python/sqlalchemy-bigquery/latest -.. _Product Documentation: https://cloud.google.com/bigquery/docs/ - +.. _SQLAlchemy dialect for BigQuery API: +.. _Client Library Documentation: https://cloud.google.com/python/docs/reference//latest +.. _Product Documentation: Quick Start ----------- @@ -25,17 +24,17 @@ Quick Start In order to use this library, you first need to go through the following steps: 1. `Select or create a Cloud Platform project.`_ -2. [Optional] `Enable billing for your project.`_ -3. `Enable the BigQuery Storage API.`_ +2. `Enable billing for your project.`_ +3. `Enable the SQLAlchemy dialect for BigQuery API.`_ 4. `Setup Authentication.`_ .. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project .. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Enable the BigQuery Storage API.: https://console.cloud.google.com/apis/library/bigquery.googleapis.com +.. _Enable the SQLAlchemy dialect for BigQuery API.: .. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html Installation ------------- +~~~~~~~~~~~~ Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to create isolated Python environments. The basic problem it addresses is one of @@ -48,13 +47,25 @@ dependencies. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ +Code samples and snippets +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Code samples and snippets live in the `samples/` folder. + + Supported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^ -Python >= 3.6 +Our client libraries are compatible with all current [active](https://devguide.python.org/devcycle/#in-development-main-branch) and [maintenance](https://devguide.python.org/devcycle/#maintenance-branches) versions of +Python. + +Python >= 3.7 Unsupported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python <= 3.5. +Python <= 3.6 + +If you are using an [end-of-life](https://devguide.python.org/devcycle/#end-of-life-branches) +version of Python, we recommend that you update as soon as possible to an actively supported version. Mac/Linux @@ -78,185 +89,15 @@ Windows \Scripts\activate \Scripts\pip.exe install sqlalchemy-bigquery -Usage ------ - -SQLAlchemy -^^^^^^^^^^ - -.. code-block:: python - - from sqlalchemy import * - from sqlalchemy.engine import create_engine - from sqlalchemy.schema import * - engine = create_engine('bigquery://project') - table = Table('dataset.table', MetaData(bind=engine), autoload=True) - print(select([func.count('*')], from_obj=table).scalar()) - -Project -^^^^^^^ - -``project`` in ``bigquery://project`` is used to instantiate BigQuery client with the specific project ID. To infer project from the environment, use ``bigquery://`` – without ``project`` - -Authentication -^^^^^^^^^^^^^^ - -Follow the `Google Cloud library guide `_ for authentication. Alternatively, you can provide the path to a service account JSON file in ``create_engine()``: - -.. code-block:: python - - engine = create_engine('bigquery://', credentials_path='/path/to/keyfile.json') - - -Location -^^^^^^^^ - -To specify location of your datasets pass ``location`` to ``create_engine()``: - -.. code-block:: python - - engine = create_engine('bigquery://project', location="asia-northeast1") - - -Table names -^^^^^^^^^^^ - -To query tables from non-default projects or datasets, use the following format for the SQLAlchemy schema name: ``[project.]dataset``, e.g.: - -.. code-block:: python - - # If neither dataset nor project are the default - sample_table_1 = Table('natality', schema='bigquery-public-data.samples') - # If just dataset is not the default - sample_table_2 = Table('natality', schema='bigquery-public-data') - -Batch size -^^^^^^^^^^ - -By default, ``arraysize`` is set to ``5000``. ``arraysize`` is used to set the batch size for fetching results. To change it, pass ``arraysize`` to ``create_engine()``: - -.. code-block:: python - - engine = create_engine('bigquery://project', arraysize=1000) - -Page size for dataset.list_tables -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, ``list_tables_page_size`` is set to ``1000``. ``list_tables_page_size`` is used to set the max_results for `dataset.list_tables`_ operation. To change it, pass ``list_tables_page_size`` to ``create_engine()``: - -.. _`dataset.list_tables`: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list -.. code-block:: python - - engine = create_engine('bigquery://project', list_tables_page_size=100) - -Adding a Default Dataset -^^^^^^^^^^^^^^^^^^^^^^^^ - -If you want to have the ``Client`` use a default dataset, specify it as the "database" portion of the connection string. - -.. code-block:: python - - engine = create_engine('bigquery://project/dataset') - -When using a default dataset, don't include the dataset name in the table name, e.g.: - -.. code-block:: python - - table = Table('table_name') - -Note that specifying a default dataset doesn't restrict execution of queries to that particular dataset when using raw queries, e.g.: - -.. code-block:: python - - # Set default dataset to dataset_a - engine = create_engine('bigquery://project/dataset_a') - - # This will still execute and return rows from dataset_b - engine.execute('SELECT * FROM dataset_b.table').fetchall() - - -Connection String Parameters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are many situations where you can't call ``create_engine`` directly, such as when using tools like `Flask SQLAlchemy `_. For situations like these, or for situations where you want the ``Client`` to have a `default_query_job_config `_, you can pass many arguments in the query of the connection string. - -The ``credentials_path``, ``credentials_info``, ``credentials_base64``, ``location``, ``arraysize`` and ``list_tables_page_size`` parameters are used by this library, and the rest are used to create a `QueryJobConfig `_ - -Note that if you want to use query strings, it will be more reliable if you use three slashes, so ``'bigquery:///?a=b'`` will work reliably, but ``'bigquery://?a=b'`` might be interpreted as having a "database" of ``?a=b``, depending on the system being used to parse the connection string. - -Here are examples of all the supported arguments. Any not present are either for legacy sql (which isn't supported by this library), or are too complex and are not implemented. - -.. code-block:: python - - engine = create_engine( - 'bigquery://some-project/some-dataset' '?' - 'credentials_path=/some/path/to.json' '&' - 'location=some-location' '&' - 'arraysize=1000' '&' - 'list_tables_page_size=100' '&' - 'clustering_fields=a,b,c' '&' - 'create_disposition=CREATE_IF_NEEDED' '&' - 'destination=different-project.different-dataset.table' '&' - 'destination_encryption_configuration=some-configuration' '&' - 'dry_run=true' '&' - 'labels=a:b,c:d' '&' - 'maximum_bytes_billed=1000' '&' - 'priority=INTERACTIVE' '&' - 'schema_update_options=ALLOW_FIELD_ADDITION,ALLOW_FIELD_RELAXATION' '&' - 'use_query_cache=true' '&' - 'write_disposition=WRITE_APPEND' - ) - -In cases where you wish to include the full credentials in the connection URI you can base64 the credentials JSON file and supply the encoded string to the ``credentials_base64`` parameter. - -.. code-block:: python - - engine = create_engine( - 'bigquery://some-project/some-dataset' '?' - 'credentials_base64=eyJrZXkiOiJ2YWx1ZSJ9Cg==' '&' - 'location=some-location' '&' - 'arraysize=1000' '&' - 'list_tables_page_size=100' '&' - 'clustering_fields=a,b,c' '&' - 'create_disposition=CREATE_IF_NEEDED' '&' - 'destination=different-project.different-dataset.table' '&' - 'destination_encryption_configuration=some-configuration' '&' - 'dry_run=true' '&' - 'labels=a:b,c:d' '&' - 'maximum_bytes_billed=1000' '&' - 'priority=INTERACTIVE' '&' - 'schema_update_options=ALLOW_FIELD_ADDITION,ALLOW_FIELD_RELAXATION' '&' - 'use_query_cache=true' '&' - 'write_disposition=WRITE_APPEND' - ) - -To create the base64 encoded string you can use the command line tool ``base64``, or ``openssl base64``, or ``python -m base64``. - -Alternatively, you can use an online generator like `www.base64encode.org _` to paste your credentials JSON file to be encoded. - -Creating tables -^^^^^^^^^^^^^^^ - -To add metadata to a table: - -.. code-block:: python - - table = Table('mytable', ..., bigquery_description='my table description', bigquery_friendly_name='my table friendly name') - -To add metadata to a column: - -.. code-block:: python - - Column('mycolumn', doc='my column description') - - -Threading and Multiprocessing -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Next Steps +~~~~~~~~~~ -Because this client uses the `grpc` library, it's safe to -share instances across threads. +- Read the `Client Library Documentation`_ for SQLAlchemy dialect for BigQuery API + to see other available methods on the client. +- Read the `SQLAlchemy dialect for BigQuery API Product documentation`_ to learn + more about the product and see How-to Guides. +- View this `README`_ to see the full list of Cloud + APIs that we cover. -In multiprocessing scenarios, the best -practice is to create client instances *after* the invocation of -`os.fork` by `multiprocessing.pool.Pool` or -`multiprocessing.Process`. +.. _SQLAlchemy dialect for BigQuery API Product documentation: +.. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst diff --git a/noxfile.py b/noxfile.py index 8a2b685e..6195ff06 100644 --- a/noxfile.py +++ b/noxfile.py @@ -20,6 +20,7 @@ import os import pathlib import re +import re import shutil import warnings @@ -477,3 +478,90 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def prerelease_deps(session): + """Run all tests with prerelease versions of dependencies installed.""" + + # Install all dependencies + session.install("-e", ".[all, tests, tracing]") + session.install(*UNIT_TEST_STANDARD_DEPENDENCIES) + system_deps_all = ( + SYSTEM_TEST_STANDARD_DEPENDENCIES + + SYSTEM_TEST_EXTERNAL_DEPENDENCIES + + SYSTEM_TEST_EXTRAS + ) + session.install(*system_deps_all) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + session.install(*constraints_deps) + + if os.path.exists("samples/snippets/requirements.txt"): + session.install("-r", "samples/snippets/requirements.txt") + + if os.path.exists("samples/snippets/requirements-test.txt"): + session.install("-r", "samples/snippets/requirements-test.txt") + + prerel_deps = [ + "protobuf", + # dependency of grpc + "six", + "googleapis-common-protos", + "grpcio", + "grpcio-status", + "google-api-core", + "proto-plus", + "google-cloud-testutils", + # dependencies of google-cloud-testutils" + "click", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + "google-auth", + ] + session.install(*other_deps) + + # Print out prerelease package versions + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run("python", "-c", "import grpc; print(grpc.__version__)") + + session.run("py.test", "tests/unit") + + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Only run system tests if found. + if os.path.exists(system_test_path) or os.path.exists(system_test_folder_path): + session.run("py.test", "tests/system") + + snippets_test_path = os.path.join("samples", "snippets") + + # Only run samples tests if found. + if os.path.exists(snippets_test_path): + session.run("py.test", "samples/snippets") diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index a40410b5..29b5bc85 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index 275d6498..6f069c6c 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.6+. +#. Create a virtualenv. Samples are compatible with Python 3.7+. .. code-block:: bash From 064344c7fbc1d5e979b5f08735874a0cb8404299 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 7 Jul 2022 16:00:56 +0000 Subject: [PATCH 2/6] require python 3.7+ in setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 7a3fe9ce..56eb0340 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,6 @@ def readme(): "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", From f9d5d43806334be0a9838804eae2b4e9b5c039b7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 7 Jul 2022 16:01:30 +0000 Subject: [PATCH 3/6] remove python 3.6 sample configs --- .kokoro/samples/python3.6/common.cfg | 40 --------------------- .kokoro/samples/python3.6/continuous.cfg | 7 ---- .kokoro/samples/python3.6/periodic-head.cfg | 11 ------ .kokoro/samples/python3.6/periodic.cfg | 6 ---- .kokoro/samples/python3.6/presubmit.cfg | 6 ---- 5 files changed, 70 deletions(-) delete mode 100644 .kokoro/samples/python3.6/common.cfg delete mode 100644 .kokoro/samples/python3.6/continuous.cfg delete mode 100644 .kokoro/samples/python3.6/periodic-head.cfg delete mode 100644 .kokoro/samples/python3.6/periodic.cfg delete mode 100644 .kokoro/samples/python3.6/presubmit.cfg diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg deleted file mode 100644 index 5cfb556d..00000000 --- a/.kokoro/samples/python3.6/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.6" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py36" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-sqlalchemy/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-sqlalchemy/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg deleted file mode 100644 index 7218af14..00000000 --- a/.kokoro/samples/python3.6/continuous.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg deleted file mode 100644 index abf3481d..00000000 --- a/.kokoro/samples/python3.6/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-sqlalchemy/.kokoro/test-samples-against-head.sh" -} diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg deleted file mode 100644 index 71cd1e59..00000000 --- a/.kokoro/samples/python3.6/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg deleted file mode 100644 index a1c8d975..00000000 --- a/.kokoro/samples/python3.6/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file From 5992f8f68ca1117af4ef9d42de31d5c5ab699222 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Sun, 10 Jul 2022 12:26:12 +0000 Subject: [PATCH 4/6] exclude templated README --- README.rst | 233 ++++++++++++++++++++++++++++++++++++++++++++--------- owlbot.py | 2 +- 2 files changed, 197 insertions(+), 38 deletions(-) diff --git a/README.rst b/README.rst index 5675a6c9..0a426ac3 100644 --- a/README.rst +++ b/README.rst @@ -1,22 +1,23 @@ -Python Client for SQLAlchemy dialect for BigQuery API -===================================================== +SQLAlchemy Dialect for BigQuery +=============================== -|preview| |pypi| |versions| +|GA| |pypi| |versions| -`SQLAlchemy dialect for BigQuery API`_: +`SQLALchemy Dialects`_ -- `Client Library Documentation`_ +- `Dialect Documentation`_ - `Product Documentation`_ -.. |preview| image:: https://img.shields.io/badge/support-preview-orange.svg - :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels +.. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability .. |pypi| image:: https://img.shields.io/pypi/v/sqlalchemy-bigquery.svg :target: https://pypi.org/project/sqlalchemy-bigquery/ .. |versions| image:: https://img.shields.io/pypi/pyversions/sqlalchemy-bigquery.svg :target: https://pypi.org/project/sqlalchemy-bigquery/ -.. _SQLAlchemy dialect for BigQuery API: -.. _Client Library Documentation: https://cloud.google.com/python/docs/reference//latest -.. _Product Documentation: +.. _SQLAlchemy Dialects: https://docs.sqlalchemy.org/en/14/dialects/ +.. _Dialect Documentation: https://googleapis.dev/python/sqlalchemy-bigquery/latest +.. _Product Documentation: https://cloud.google.com/bigquery/docs/ + Quick Start ----------- @@ -24,17 +25,17 @@ Quick Start In order to use this library, you first need to go through the following steps: 1. `Select or create a Cloud Platform project.`_ -2. `Enable billing for your project.`_ -3. `Enable the SQLAlchemy dialect for BigQuery API.`_ +2. [Optional] `Enable billing for your project.`_ +3. `Enable the BigQuery Storage API.`_ 4. `Setup Authentication.`_ .. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project .. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Enable the SQLAlchemy dialect for BigQuery API.: +.. _Enable the BigQuery Storage API.: https://console.cloud.google.com/apis/library/bigquery.googleapis.com .. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html Installation -~~~~~~~~~~~~ +------------ Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to create isolated Python environments. The basic problem it addresses is one of @@ -47,25 +48,13 @@ dependencies. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ -Code samples and snippets -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Code samples and snippets live in the `samples/` folder. - - Supported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^ -Our client libraries are compatible with all current [active](https://devguide.python.org/devcycle/#in-development-main-branch) and [maintenance](https://devguide.python.org/devcycle/#maintenance-branches) versions of -Python. - Python >= 3.7 Unsupported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python <= 3.6 - -If you are using an [end-of-life](https://devguide.python.org/devcycle/#end-of-life-branches) -version of Python, we recommend that you update as soon as possible to an actively supported version. +Python <= 3.6. Mac/Linux @@ -89,15 +78,185 @@ Windows \Scripts\activate \Scripts\pip.exe install sqlalchemy-bigquery -Next Steps -~~~~~~~~~~ +Usage +----- + +SQLAlchemy +^^^^^^^^^^ + +.. code-block:: python + + from sqlalchemy import * + from sqlalchemy.engine import create_engine + from sqlalchemy.schema import * + engine = create_engine('bigquery://project') + table = Table('dataset.table', MetaData(bind=engine), autoload=True) + print(select([func.count('*')], from_obj=table).scalar()) + +Project +^^^^^^^ + +``project`` in ``bigquery://project`` is used to instantiate BigQuery client with the specific project ID. To infer project from the environment, use ``bigquery://`` – without ``project`` + +Authentication +^^^^^^^^^^^^^^ + +Follow the `Google Cloud library guide `_ for authentication. Alternatively, you can provide the path to a service account JSON file in ``create_engine()``: + +.. code-block:: python + + engine = create_engine('bigquery://', credentials_path='/path/to/keyfile.json') + + +Location +^^^^^^^^ + +To specify location of your datasets pass ``location`` to ``create_engine()``: + +.. code-block:: python + + engine = create_engine('bigquery://project', location="asia-northeast1") + + +Table names +^^^^^^^^^^^ + +To query tables from non-default projects or datasets, use the following format for the SQLAlchemy schema name: ``[project.]dataset``, e.g.: + +.. code-block:: python + + # If neither dataset nor project are the default + sample_table_1 = Table('natality', schema='bigquery-public-data.samples') + # If just dataset is not the default + sample_table_2 = Table('natality', schema='bigquery-public-data') + +Batch size +^^^^^^^^^^ + +By default, ``arraysize`` is set to ``5000``. ``arraysize`` is used to set the batch size for fetching results. To change it, pass ``arraysize`` to ``create_engine()``: + +.. code-block:: python + + engine = create_engine('bigquery://project', arraysize=1000) + +Page size for dataset.list_tables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, ``list_tables_page_size`` is set to ``1000``. ``list_tables_page_size`` is used to set the max_results for `dataset.list_tables`_ operation. To change it, pass ``list_tables_page_size`` to ``create_engine()``: + +.. _`dataset.list_tables`: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list +.. code-block:: python + + engine = create_engine('bigquery://project', list_tables_page_size=100) + +Adding a Default Dataset +^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to have the ``Client`` use a default dataset, specify it as the "database" portion of the connection string. + +.. code-block:: python + + engine = create_engine('bigquery://project/dataset') + +When using a default dataset, don't include the dataset name in the table name, e.g.: + +.. code-block:: python + + table = Table('table_name') + +Note that specifying a default dataset doesn't restrict execution of queries to that particular dataset when using raw queries, e.g.: + +.. code-block:: python + + # Set default dataset to dataset_a + engine = create_engine('bigquery://project/dataset_a') + + # This will still execute and return rows from dataset_b + engine.execute('SELECT * FROM dataset_b.table').fetchall() + + +Connection String Parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are many situations where you can't call ``create_engine`` directly, such as when using tools like `Flask SQLAlchemy `_. For situations like these, or for situations where you want the ``Client`` to have a `default_query_job_config `_, you can pass many arguments in the query of the connection string. + +The ``credentials_path``, ``credentials_info``, ``credentials_base64``, ``location``, ``arraysize`` and ``list_tables_page_size`` parameters are used by this library, and the rest are used to create a `QueryJobConfig `_ + +Note that if you want to use query strings, it will be more reliable if you use three slashes, so ``'bigquery:///?a=b'`` will work reliably, but ``'bigquery://?a=b'`` might be interpreted as having a "database" of ``?a=b``, depending on the system being used to parse the connection string. + +Here are examples of all the supported arguments. Any not present are either for legacy sql (which isn't supported by this library), or are too complex and are not implemented. + +.. code-block:: python + + engine = create_engine( + 'bigquery://some-project/some-dataset' '?' + 'credentials_path=/some/path/to.json' '&' + 'location=some-location' '&' + 'arraysize=1000' '&' + 'list_tables_page_size=100' '&' + 'clustering_fields=a,b,c' '&' + 'create_disposition=CREATE_IF_NEEDED' '&' + 'destination=different-project.different-dataset.table' '&' + 'destination_encryption_configuration=some-configuration' '&' + 'dry_run=true' '&' + 'labels=a:b,c:d' '&' + 'maximum_bytes_billed=1000' '&' + 'priority=INTERACTIVE' '&' + 'schema_update_options=ALLOW_FIELD_ADDITION,ALLOW_FIELD_RELAXATION' '&' + 'use_query_cache=true' '&' + 'write_disposition=WRITE_APPEND' + ) + +In cases where you wish to include the full credentials in the connection URI you can base64 the credentials JSON file and supply the encoded string to the ``credentials_base64`` parameter. + +.. code-block:: python + + engine = create_engine( + 'bigquery://some-project/some-dataset' '?' + 'credentials_base64=eyJrZXkiOiJ2YWx1ZSJ9Cg==' '&' + 'location=some-location' '&' + 'arraysize=1000' '&' + 'list_tables_page_size=100' '&' + 'clustering_fields=a,b,c' '&' + 'create_disposition=CREATE_IF_NEEDED' '&' + 'destination=different-project.different-dataset.table' '&' + 'destination_encryption_configuration=some-configuration' '&' + 'dry_run=true' '&' + 'labels=a:b,c:d' '&' + 'maximum_bytes_billed=1000' '&' + 'priority=INTERACTIVE' '&' + 'schema_update_options=ALLOW_FIELD_ADDITION,ALLOW_FIELD_RELAXATION' '&' + 'use_query_cache=true' '&' + 'write_disposition=WRITE_APPEND' + ) + +To create the base64 encoded string you can use the command line tool ``base64``, or ``openssl base64``, or ``python -m base64``. + +Alternatively, you can use an online generator like `www.base64encode.org _` to paste your credentials JSON file to be encoded. + +Creating tables +^^^^^^^^^^^^^^^ + +To add metadata to a table: + +.. code-block:: python + + table = Table('mytable', ..., bigquery_description='my table description', bigquery_friendly_name='my table friendly name') + +To add metadata to a column: + +.. code-block:: python + + Column('mycolumn', doc='my column description') + + +Threading and Multiprocessing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Read the `Client Library Documentation`_ for SQLAlchemy dialect for BigQuery API - to see other available methods on the client. -- Read the `SQLAlchemy dialect for BigQuery API Product documentation`_ to learn - more about the product and see How-to Guides. -- View this `README`_ to see the full list of Cloud - APIs that we cover. +Because this client uses the `grpc` library, it's safe to +share instances across threads. -.. _SQLAlchemy dialect for BigQuery API Product documentation: -.. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst +In multiprocessing scenarios, the best +practice is to create client instances *after* the invocation of +`os.fork` by `multiprocessing.pool.Pool` or +`multiprocessing.Process`. diff --git a/owlbot.py b/owlbot.py index cd3a7226..df417029 100644 --- a/owlbot.py +++ b/owlbot.py @@ -33,7 +33,6 @@ "3.10": ["tests", "geography"], } templated_files = common.py_library( - unit_test_python_versions=["3.6", "3.7", "3.8", "3.9", "3.10"], system_test_python_versions=["3.8", "3.10"], cov_level=100, unit_test_extras=extras, @@ -47,6 +46,7 @@ "docs/multiprocessing.rst", # exclude gh actions as credentials are needed for tests ".github/workflows", + "README.rst", ]) # ---------------------------------------------------------------------------- From 4db719d2f5d3d9ed14aaaa5740ab860e562ef4fe Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Sun, 10 Jul 2022 12:28:01 +0000 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- CONTRIBUTING.rst | 6 ++---- noxfile.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 3a6be172..3c6795e8 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.6, 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. + 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -221,13 +221,11 @@ Supported Python Versions We support: -- `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ -.. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ @@ -239,7 +237,7 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-bigquery-sqlalchemy/blob/main/noxfile.py -We also explicitly decided to support Python 3 beginning with version 3.6. +We also explicitly decided to support Python 3 beginning with version 3.7. Reasons for this include: - Encouraging use of newest versions of Python 3 diff --git a/noxfile.py b/noxfile.py index 6195ff06..5f3deae9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -32,7 +32,7 @@ DEFAULT_PYTHON_VERSION = "3.8" -UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", From 1a82100cddb798c5df01be3f545037a4a4f0efb9 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Sun, 10 Jul 2022 13:26:45 -0400 Subject: [PATCH 6/6] update python_requires in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 56eb0340..4afe003e 100644 --- a/setup.py +++ b/setup.py @@ -95,7 +95,7 @@ def readme(): "future", ], extras_require=extras, - python_requires=">=3.6, <3.11", + python_requires=">=3.7, <3.11", tests_require=["packaging", "pytz"], entry_points={ "sqlalchemy.dialects": ["bigquery = sqlalchemy_bigquery:BigQueryDialect"]