fix: properly exclude from indexes non-indexed subproperties of structured properties (#346)

cguardia · web-flow · commit ae8f53348154 · 2020-02-26T14:24:10.000-06:00
Refs (#341).
diff --git a/packages/google-cloud-ndb/google/cloud/ndb/model.py b/packages/google-cloud-ndb/google/cloud/ndb/model.py
@@ -724,26 +724,22 @@ def _entity_to_ds_entity(entity, set_key=True):
     Raises:
         ndb.exceptions.BadValueError: If entity has uninitialized properties.
     """
-    data = {}
+    data = {"_exclude_from_indexes": []}
     uninitialized = []
-    exclude_from_indexes = []
 
     for prop in _properties_of(entity):
         if not prop._is_initialized(entity):
             uninitialized.append(prop._name)
 
-        names = prop._to_datastore(entity, data)
-
-        if not prop._indexed:
-            for name in names:
-                exclude_from_indexes.append(name)
+        prop._to_datastore(entity, data)
 
     if uninitialized:
         missing = ", ".join(uninitialized)
         raise exceptions.BadValueError(
             "Entity has uninitialized properties: {}".format(missing)
         )
 
+    exclude_from_indexes = data.pop("_exclude_from_indexes")
     ds_entity = None
     if set_key:
         key = entity._key
@@ -2095,6 +2091,9 @@ def _to_datastore(self, entity, data, prefix="", repeated=False):
         else:
             data[key] = value
 
+        if not self._indexed:
+            data["_exclude_from_indexes"].append(key)
+
         return (key,)
 
     def _from_datastore(self, ds_entity, value):
diff --git a/packages/google-cloud-ndb/tests/system/test_crud.py b/packages/google-cloud-ndb/tests/system/test_crud.py
@@ -1254,3 +1254,26 @@ class SomeKind(ndb.Model):
 
         assert isinstance(retrieved.bar[0], OtherKind)
         assert isinstance(retrieved.bar[1], OtherKind)
+
+
+def test_insert_structured_property_with_unindexed_subproperty_legacy_data(
+    client_context, dispose_of, ds_client
+):
+    """Regression test for #341
+
+    https://github.com/googleapis/python-ndb/issues/341
+    """
+
+    class OtherKind(ndb.Model):
+        data = ndb.BlobProperty(indexed=False)
+
+    class SomeKind(ndb.Model):
+        entry = ndb.StructuredProperty(OtherKind)
+
+    with client_context.new(legacy_data=True).use():
+        entity = SomeKind(entry=OtherKind(data=b"01234567890" * 1000))
+        key = entity.put()
+        dispose_of(key._key)
+
+        retrieved = key.get()
+        assert isinstance(retrieved.entry, OtherKind)
diff --git a/packages/google-cloud-ndb/tests/unit/test_model.py b/packages/google-cloud-ndb/tests/unit/test_model.py
@@ -3476,6 +3476,21 @@ class SomeKind(model.Model):
             assert SomeKind.foo._to_datastore(entity, data) == {"foo"}
             assert data == {"foo": None}
 
+    @staticmethod
+    def test__to_datastore_legacy_subentity_is_unindexed(in_context):
+        class SubKind(model.Model):
+            bar = model.BlobProperty(indexed=False)
+
+        class SomeKind(model.Model):
+            foo = model.StructuredProperty(SubKind)
+
+        with in_context.new(legacy_data=True).use():
+            entity = SomeKind(foo=SubKind())
+            data = {"_exclude_from_indexes": []}
+            assert SomeKind.foo._to_datastore(entity, data) == {"foo.bar"}
+            assert data.pop("_exclude_from_indexes") == ["foo.bar"]
+            assert data == {"foo.bar": None}
+
     @staticmethod
     def test__to_datastore_legacy_repeated(in_context):
         class SubKind(model.Model):
@@ -3682,12 +3697,13 @@ class SomeKind(model.Model):
             )
 
         entity = SomeKind(foo=[SubKind(bar="baz")])
-        data = {}
+        data = {"_exclude_from_indexes": []}
         protobuf = model._entity_to_protobuf(entity.foo[0], set_key=False)
         protobuf = protobuf.SerializePartialToString()
         assert SomeKind.foo._to_datastore(entity, data, repeated=True) == (
             "foo",
         )
+        assert data.pop("_exclude_from_indexes") == ["foo"]
         assert data == {"foo": [[protobuf]]}
 
     @staticmethod
@@ -3702,13 +3718,14 @@ class SomeKind(model.Model):
 
         with in_context.new(legacy_data=True).use():
             entity = SomeKind(foo=[SubKind(bar="baz")])
-            data = {}
+            data = {"_exclude_from_indexes": []}
             ds_entity = model._entity_to_ds_entity(
                 entity.foo[0], set_key=False
             )
             assert SomeKind.foo._to_datastore(entity, data, repeated=True) == (
                 "foo",
             )
+            assert data.pop("_exclude_from_indexes") == ["foo"]
             assert data == {"foo": [ds_entity]}
 
     @staticmethod
@@ -3721,8 +3738,9 @@ class SomeKind(model.Model):
 
         with in_context.new(legacy_data=True).use():
             entity = SomeKind(foo=SubKind(bar="baz"))
-            data = {}
+            data = {"_exclude_from_indexes": []}
             assert SomeKind.foo._to_datastore(entity, data) == ("foo",)
+            assert data.pop("_exclude_from_indexes") == ["foo"]
             ds_entity = model._entity_to_ds_entity(entity.foo, set_key=False)
             assert data == {"foo": ds_entity}
 
