googleapis · IlyaFaer · Nov 26, 2020 · Nov 30, 2020 · Dec 1, 2020 · Dec 3, 2020
diff --git a/.github/workflows/django_tests_against_emulator.yml b/.github/workflows/django_tests_against_emulator.yml
@@ -0,0 +1,61 @@
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+name: django-tests
+jobs:
+  system-tests:
+    runs-on: ubuntu-latest
+
+    services:
+      emulator-0:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9010:9010
+      emulator-1:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9011:9010
+      emulator-2:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9012:9010
+      emulator-3:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9013:9010
+      emulator-4:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9014:9010
+      emulator-5:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9015:9010
+      emulator-6:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9016:9010
+      emulator-7:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9017:9010
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Run Django tests
+        run: sh django_test_suite.sh
+        env:
+          SPANNER_EMULATOR_HOST: localhost:9010
+          GOOGLE_CLOUD_PROJECT: emulator-test-project
+          GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true
+          RUNNING_SPANNER_BACKEND_TESTS: 1
+          DJANGO_WORKER_INDEX: 0
+          DJANGO_WORKER_COUNT: 1
+          SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests
diff --git a/create_test_instance.py b/create_test_instance.py
@@ -0,0 +1,26 @@
+# Copyright 2016 Google LLC All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from google.cloud.spanner_v1 import Client
+
+
+client = Client(
+    project=os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project")
+)
+
+instance = client.instance("google-cloud-django-backend-tests")
+created_op = instance.create()
+created_op.result(30)  # block until completion
diff --git a/django_spanner/base.py b/django_spanner/base.py
@@ -4,6 +4,8 @@
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
 
+import os
+
 from django.db.backends.base.base import BaseDatabaseWrapper
 from google.cloud import spanner_dbapi as Database, spanner_v1 as spanner
 
@@ -103,7 +105,9 @@ class DatabaseWrapper(BaseDatabaseWrapper):
 
     @property
     def instance(self):
-        return spanner.Client().instance(self.settings_dict["INSTANCE"])
+        return spanner.Client(
+            project=os.environ["GOOGLE_CLOUD_PROJECT"]
+        ).instance(self.settings_dict["INSTANCE"])
 
     @property
     def _nodb_connection(self):
@@ -113,7 +117,7 @@ def _nodb_connection(self):
 
     def get_connection_params(self):
         return {
-            "project": self.settings_dict["PROJECT"],
+            "project": os.environ["GOOGLE_CLOUD_PROJECT"],
             "instance_id": self.settings_dict["INSTANCE"],
             "database_id": self.settings_dict["NAME"],
             "user_agent": "django_spanner/2.2.0a1",

diff --git a/django_spanner/features.py b/django_spanner/features.py
@@ -21,7 +21,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     # Spanner uses REGEXP_CONTAINS which is case-sensitive.
     has_case_insensitive_like = False
     # https://cloud.google.com/spanner/quotas#query_limits
-    max_query_params = 950
+    max_query_params = 900
     supports_foreign_keys = False
     supports_ignore_conflicts = False
     supports_partial_indexes = False
@@ -326,17 +326,13 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         "model_formsets.tests.ModelFormsetTest.test_prevent_change_outer_model_and_create_invalid_data",
         "model_formsets_regress.tests.FormfieldShouldDeleteFormTests.test_no_delete",
         "model_formsets_regress.tests.FormsetTests.test_extraneous_query_is_not_run",
+        # os.chmod() doesn't work on Kokoro?
+        "file_uploads.tests.DirectoryCreationTests.test_readonly_root",
+        # Tests that sometimes fail on Kokoro for unknown reasons.
+        "contenttypes_tests.test_models.ContentTypesTests.test_cache_not_shared_between_managers",
+        "migration_test_data_persistence.tests.MigrationDataNormalPersistenceTestCase.test_persistence",
+        "servers.test_liveserverthread.LiveServerThreadTest.test_closes_connections",
     )
-    # Kokoro-specific skips.
-    if os.environ.get("KOKORO_JOB_NAME"):
-        skip_tests += (
-            # os.chmod() doesn't work on Kokoro?
-            "file_uploads.tests.DirectoryCreationTests.test_readonly_root",
-            # Tests that sometimes fail on Kokoro for unknown reasons.
-            "contenttypes_tests.test_models.ContentTypesTests.test_cache_not_shared_between_managers",
-            "migration_test_data_persistence.tests.MigrationDataNormalPersistenceTestCase.test_persistence",
-            "servers.test_liveserverthread.LiveServerThreadTest.test_closes_connections",
-        )
 
     if os.environ.get("SPANNER_EMULATOR_HOST", None):
         # Some code isn't yet supported by the Spanner emulator.
@@ -1106,8 +1102,28 @@ class DatabaseFeatures(BaseDatabaseFeatures):
             "expressions.tests.ValueTests.test_update_TimeField_using_Value",  # noqa
             "expressions.tests.ValueTests.test_update_UUIDField_using_Value",  # noqa
             "fixtures.tests.FixtureLoadingTests.test_loaddata_error_message",  # noqa
+            "fixtures.tests.FixtureTransactionTests.test_format_discovery",  # noqa
             "fixtures.tests.ForwardReferenceTests.test_forward_reference_fk",  # noqa
             "fixtures.tests.ForwardReferenceTests.test_forward_reference_m2m",  # noqa
+            "flatpages_tests.test_csrf.FlatpageCSRFTests.test_view_authenticated_flatpage",  # noqa
+            "flatpages_tests.test_middleware.FlatpageMiddlewareTests.test_fallback_authenticated_flatpage",  # noqa
+            "flatpages_tests.test_middleware.FlatpageMiddlewareTests.test_view_authenticated_flatpage",  # noqa
+            "flatpages_tests.test_templatetags.FlatpageTemplateTagTests.test_get_flatpages_tag_for_user",  # noqa
+            "flatpages_tests.test_templatetags.FlatpageTemplateTagTests.test_get_flatpages_with_prefix_for_user",  # noqa
+            "flatpages_tests.test_views.FlatpageViewTests.test_view_authenticated_flatpage",  # noqa
+            "generic_inline_admin.tests.GenericAdminViewTest.test_basic_add_GET",  # noqa
+            "generic_inline_admin.tests.GenericAdminViewTest.test_basic_add_POST",  # noqa
+            "generic_inline_admin.tests.GenericAdminViewTest.test_basic_edit_GET",  # noqa
+            "generic_inline_admin.tests.GenericAdminViewTest.test_basic_edit_POST",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminParametersTest.testMaxNumParam",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_get_extra",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_extra_param",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_get_max_num",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_get_min_num",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_min_num_param",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_no_param",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminWithUniqueTogetherTest.test_add",  # noqa
+            "generic_inline_admin.tests.GenericInlineAdminWithUniqueTogetherTest.test_delete",  # noqa
             "get_or_create.tests.GetOrCreateTests.test_get_or_create_invalid_params",  # noqa
             "get_or_create.tests.GetOrCreateTestsWithManualPKs.test_create_with_duplicate_primary_key",  # noqa
             "get_or_create.tests.GetOrCreateTestsWithManualPKs.test_get_or_create_raises_IntegrityError_plus_traceback",  # noqa

diff --git a/django_test_suite.sh b/django_test_suite.sh
@@ -4,8 +4,37 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-# exit when any command fails
-set -e
+set -x pipefail
+
+sudo apt-get update -y
+sudo apt-get install -y libmemcached-dev
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+pip3 install .
+pip3 uninstall -y google-cloud-spanner
+pip3 uninstall -y django-google-spanner
+pip3 install -e 'git+https://github.com/q-logic/python-spanner.git@autocommit_change#egg=google-cloud-spanner'
+pip3 install -e 'git+https://github.com/q-logic/python-spanner-django.git@dj_tests_against_emulator#egg=django-google-spanner'
+
+export DJANGO_TESTS_DIR="django_tests_dir"
+mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch spanner-2.2.x https://github.com/timgraham/django.git $DJANGO_TESTS_DIR/django
+
+# Install dependencies for Django tests.
+sudo apt-get update
+sudo apt-get install -y libffi-dev libjpeg-dev zlib1g-devel
+
+cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
+
+SPANNER_EMULATOR_HOST=localhost:9010 python3 create_test_instance.py
+SPANNER_EMULATOR_HOST=localhost:9011 python3 create_test_instance.py
+SPANNER_EMULATOR_HOST=localhost:9012 python3 create_test_instance.py
+SPANNER_EMULATOR_HOST=localhost:9013 python3 create_test_instance.py
+SPANNER_EMULATOR_HOST=localhost:9014 python3 create_test_instance.py
+SPANNER_EMULATOR_HOST=localhost:9015 python3 create_test_instance.py
+SPANNER_EMULATOR_HOST=localhost:9016 python3 create_test_instance.py
+SPANNER_EMULATOR_HOST=localhost:9017 python3 create_test_instance.py
 
 # If no SPANNER_TEST_DB is set, generate a unique one
 # so that we can have multiple tests running without
@@ -15,7 +44,9 @@ TEST_DBNAME=${SPANNER_TEST_DB:-$(python3 -c 'import os, time; print(chr(ord("a")
 TEST_DBNAME_OTHER="$TEST_DBNAME-ot"
 TEST_APPS=${DJANGO_TEST_APPS:-basic}
 INSTANCE=${SPANNER_TEST_INSTANCE:-django-tests}
-PROJECT=${PROJECT_ID:-appdev-soda-spanner-staging}
+PROJECT=${PROJECT_ID}
+SPANNER_EMULATOR_HOST=${SPANNER_EMULATOR_HOST}
+GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT}
 SETTINGS_FILE="$TEST_DBNAME-settings"
 TESTS_DIR=${DJANGO_TESTS_DIR:-django_tests}
 
@@ -42,21 +73,14 @@ PASSWORD_HASHERS = [
 !
 }
 
-setup_emulator_if_needed() {
-    if [[ $SPANNER_EMULATOR_HOST != "" ]]
-    then
-        echo "Running the emulator at: $SPANNER_EMULATOR_HOST"
-        ./emulator_main --host_port "$SPANNER_EMULATOR_HOST" &
-        SPANNER_INSTANCE=$INSTANCE python3 .kokoro/ensure_instance_exists.py
-    fi
-}
-
-run_django_tests() {
-    cd $TESTS_DIR/django/tests
-    create_settings
-    echo -e "\033[32mRunning Django tests: $TEST_APPS\033[00m"
-    python3 runtests.py $TEST_APPS --verbosity=2 --noinput --settings $SETTINGS_FILE
-}
+cd $TESTS_DIR/django/tests
+create_settings
 
-setup_emulator_if_needed
-run_django_tests
+SPANNER_EMULATOR_HOST=localhost:9010 python3 runtests.py modeladmin model_fields model_forms model_formsets model_formsets_regress model_indexes model_inheritance model_inheritance_regress model_options model_package model_regress multiple_database --verbosity=3 --noinput --settings $SETTINGS_FILE &
+SPANNER_EMULATOR_HOST=localhost:9011 python3 mutually_referential nested_foreign_keys null_fk null_fk_ordering null_queries one_to_one ordering order_with_respect_to or_lookups prefetch_related proxy_model_inheritance proxy_models queries queryset_pickle raw_query redirects_tests reserved_names reverse_lookup save_delete_hooks schema --verbosity=3 --noinput --settings $SETTINGS_FILE &
+SPANNER_EMULATOR_HOST=localhost:9012 python3 runtests.py migration_test_data_persistence max_lengths migrate_signals migrations select_for_update select_related select_related_onetoone select_related_regress serializers servers sessions_tests signals --verbosity=3 --noinput --settings $SETTINGS_FILE &
+SPANNER_EMULATOR_HOST=localhost:9013 python3 sitemaps_tests sites_framework sites_tests string_lookup syndication_tests test_client test_client_regress test_runner test_utils timezones transaction_hooks transactions unmanaged_models update update_only_fields validation admin_changelist admin_docs view_tests many_to_many many_to_one many_to_one_null --verbosity=3 --noinput --settings $SETTINGS_FILE &
+SPANNER_EMULATOR_HOST=localhost:9014 python3 runtests.py admin_filters admin_inlines admin_ordering admin_utils admin_views aggregation aggregation_regress annotations auth_tests backends basic bulk_create cache choices constraints contenttypes_tests --verbosity=3 --noinput --settings $SETTINGS_FILE &
+SPANNER_EMULATOR_HOST=localhost:9015 python3 custom_columns custom_lookups custom_managers custom_methods custom_pk datatypes dates datetimes db_functions defer defer_regress delete delete_regress distinct_on_fields empty expressions expressions_case expressions_window extra_regress field_defaults file_storage file_uploads filtered_relation --verbosity=3 --noinput --settings $SETTINGS_FILE &
+SPANNER_EMULATOR_HOST=localhost:9016 python3 runtests.py fixtures fixtures_model_package fixtures_regress flatpages_tests force_insert_update foreign_object forms_tests from_db_value generic_inline_admin generic_relations generic_relations_regress --verbosity=3 --noinput --settings $SETTINGS_FILE
+SPANNER_EMULATOR_HOST=localhost:9017 python3 generic_views get_earliest_or_latest get_object_or_404 get_or_create i18n indexes inline_formsets inspectdb introspection invalid_models_tests known_related_objects lookup m2m_and_m2o m2m_intermediary m2m_multiple m2m_recursive m2m_regress m2m_signals m2m_through m2m_through_regress m2o_recursive managers_regress --verbosity=3 --noinput --settings $SETTINGS_FILE