# Copyright 2018 Braxton Mckee

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

from object_database.messages import ClientToServer, getHeartbeatInterval

from object_database.core_schema import core_schema

from object_database.view import View, Transaction, _cur_view, SerializedDatabaseValue

from object_database.identity import IdentityProducer

import object_database.keymapping as keymapping

from typed_python.SerializationContext import SerializationContext

from typed_python.Codebase import Codebase as TypedPythonCodebase

from typed_python import Alternative

import queue

import threading

import logging

import traceback

import time

from object_database.view import DisconnectedException

class Everything:

"""Singleton to mark subscription to everything in a slice."""

TransactionResult = Alternative(

"TransactionResult",

Success={},

RevisionConflict={'key': str},

Disconnected={}

)

class VersionedBase:

def _best_version_offset_for(self, version):

i = len(self.version_numbers) - 1

while i >= 0:

if self.version_numbers[i] <= version:

return i

i -= 1

return None

def isEmpty(self):

return not self.version_numbers

def validVersionIncoming(self, version_read, transaction_id):

if not self.version_numbers:

return True

top = self.version_numbers[-1]

assert transaction_id > version_read

return version_read >= top

def hasVersionInfoNewerThan(self, tid):

if not self.version_numbers:

return False

return tid < self.version_numbers[-1]

def newestValue(self):

if self.version_numbers:

return self.valueForVersion(self.version_numbers[-1])

else:

return self.valueForVersion(None)

class VersionedValue(VersionedBase):

def __init__(self):

self.version_numbers = []

self.values = []

def setVersionedValue(self, version_number, val):

self.version_numbers.append(version_number)

self.values.append(val)

def valueForVersion(self, version):

i = self._best_version_offset_for(version)

if i is None:

return None

return self.values[i]

def needsToTrack(self):

return len(self.version_numbers) != 1 or self.values[0].serializedByteRep is None

def cleanup(self, version_number):

if not self.values:

return True

while self.values and self.version_numbers[0] < version_number:

if len(self.values) == 1:

if self.values[0].serializedByteRep is None:

# this value was deleted and we can just remove this whole entry

return True

else:

self.version_numbers[0] = version_number

else:

if self.version_numbers[1] <= version_number:

self.values.pop(0)

self.version_numbers.pop(0)

else:

self.version_numbers[0] = version_number

def __repr__(self):

return "VersionedValue(ids=%s)" % (self.version_numbers,)

class VersionedSet(VersionedBase):

# values in sets are always strings

def __init__(self):

self.version_numbers = []

self.adds = []

self.removes = []

def setVersionedAddsAndRemoves(self, version, adds, removes):

assert not adds or not removes

assert adds or removes

assert isinstance(adds, set)

assert isinstance(removes, set)

if not self.version_numbers:

if removes:

adds = set(adds)

adds.difference_update(removes)

removes = set()

self.adds.append(adds)

self.removes.append(removes)

self.version_numbers.append(version)

def needsToTrack(self):

return len(self.version_numbers) != 1 or not (len(self.adds[0]) or len(self.removes[0]))

def updateVersionedAdds(self, version, adds):

if not self.version_numbers or self.version_numbers[-1] != version:

assert not self.version_numbers or self.version_numbers[-1] < version

self.setVersionedAddsAndRemoves(version, adds, set())

else:

# someone could be iterating over this set in another thread

new_last_adds = set(self.adds[-1])

new_last_adds.update(adds)

self.adds[-1] = new_last_adds

def cleanup(self, version_number):

if not self.version_numbers:

return True

while self.version_numbers and self.version_numbers[0] < version_number:

if len(self.version_numbers) == 1:

if not self.adds[0] and not self.removes[0]:

# this value was deleted and we can just remove this whole entry

return True

else:

self.version_numbers[0] = version_number

else:

if self.version_numbers[1] <= version_number:

# merge slot 0 into slot 1

# the new set should have no removes, and only adds

assert not self.removes[0], (self.adds[0], self.removes[0])

new_set = set(self.adds[0])

new_set.update(self.adds[1])

new_set.difference_update(self.removes[1])

self.adds.pop(0)

self.removes.pop(0)

self.version_numbers.pop(0)

self.adds[0] = new_set

self.removes[0] = set()

else:

self.version_numbers[0] = version_number

def valueForVersion(self, version):

ix = self._best_version_offset_for(version)

if ix is None:

return SetWithEdits(set(), set(), set())

return SetWithEdits(set(), self.adds[:ix+1], self.removes[:ix+1])

def __repr__(self):

return "VersionedSet(ids=%s, adds=%s, removes=%s)" % (self.version_numbers, self.adds, self.removes)

class SetWithEdits:

AGRESSIVELY_CHECK_SET_ADDS = False

def __init__(self, s, adds, removes):

self.s = s

self.adds = adds

self.removes = removes

def toSet(self):

res = set(self.s)

for i in range(len(self.adds)):

res.update(self.adds[i])

res.difference_update(self.removes[i])

return res

def pickAny(self, toAvoid):

removed = set()

for i in reversed(range(len(self.adds))):

if SetWithEdits.AGRESSIVELY_CHECK_SET_ADDS:

adds = set(self.adds[i])

for a in self.adds[i]:

if a not in removed and a not in toAvoid:

return a

if SetWithEdits.AGRESSIVELY_CHECK_SET_ADDS:

time.sleep(0.0001)

assert self.adds[i] == adds

removed.update(self.removes[i])

for a in self.s:

if a not in removed and a not in toAvoid:

return a

class ManyVersionedObjects:

def __init__(self):

# for each version number we have outstanding

self._version_number_refcount = {}

self._min_reffed_version_number = None

# for each version number, the set of keys that are set with it

self._version_number_objects = {}

# for each key, a VersionedValue or VersionedSet

self._versioned_objects = {}

def keycount(self):

return len(self._versioned_objects)

def versionIncref(self, version_number):

if version_number not in self._version_number_refcount:

self._version_number_refcount[version_number] = 1

if self._min_reffed_version_number is None:

self._min_reffed_version_number = version_number

else:

self._min_reffed_version_number = min(version_number, self._min_reffed_version_number)

else:

self._version_number_refcount[version_number] += 1

def versionDecref(self, version_number):

assert version_number in self._version_number_refcount

self._version_number_refcount[version_number] -= 1

assert self._version_number_refcount[version_number] >= 0

if self._version_number_refcount[version_number] == 0:

del self._version_number_refcount[version_number]

if version_number == self._min_reffed_version_number:

if not self._version_number_refcount:

self._min_reffed_version_number = None

else:

self._min_reffed_version_number = min(self._version_number_refcount)

def setForVersion(self, key, version_number):

if key in self._versioned_objects:

return self._versioned_objects[key].valueForVersion(version_number)

return SetWithEdits(set(), set(), set())

def hasDataForKey(self, key):

return key in self._versioned_objects

def valueForVersion(self, key, version_number):

return self._versioned_objects[key].valueForVersion(version_number)

def _object_has_version(self, key, version_number):

if version_number not in self._version_number_objects:

self._version_number_objects[version_number] = set()

self._version_number_objects[version_number].add(key)

def setVersionedValue(self, key, version_number, serialized_val):

self._object_has_version(key, version_number)

if key not in self._versioned_objects:

self._versioned_objects[key] = VersionedValue()

versioned = self._versioned_objects[key]

initialValue = versioned.newestValue()

versioned.setVersionedValue(version_number, SerializedDatabaseValue(serialized_val, {}))

return initialValue

def setVersionedAddsAndRemoves(self, key, version_number, adds, removes):

self._object_has_version(key, version_number)

if key not in self._versioned_objects:

self._versioned_objects[key] = VersionedSet()

if adds or removes:

self._versioned_objects[key].setVersionedAddsAndRemoves(version_number, adds, removes)

def setVersionedTailValueStringified(self, key, serialized_val):

if key not in self._versioned_objects:

self._object_has_version(key, -1)

self._versioned_objects[key] = VersionedValue()

self._versioned_objects[key].setVersionedValue(-1, SerializedDatabaseValue(serialized_val, {}))

def updateVersionedAdds(self, key, version_number, adds):

self._object_has_version(key, version_number)

if key not in self._versioned_objects:

self._versioned_objects[key] = VersionedSet()

self._versioned_objects[key].setVersionedAddsAndRemoves(version_number, adds, set())

else:

self._versioned_objects[key].updateVersionedAdds(version_number, adds)

def cleanup(self, curTransactionId):

"""Get rid of old objects we don't need to keep around and increase the min_transaction_id"""

if self._min_reffed_version_number is not None:

lowestId = min(self._min_reffed_version_number, curTransactionId)

else:

lowestId = curTransactionId

if self._version_number_objects:

while min(self._version_number_objects) < lowestId:

toCollapse = min(self._version_number_objects)

for key in self._version_number_objects[toCollapse]:

if key not in self._versioned_objects:

pass

elif self._versioned_objects[key].cleanup(lowestId):

del self._versioned_objects[key]

else:

if self._versioned_objects[key].needsToTrack():

self._object_has_version(key, lowestId)

del self._version_number_objects[toCollapse]

class TransactionListener:

def __init__(self, db, handler):

self._thread = threading.Thread(target=self._doWork)

self._thread.daemon = True

self._shouldStop = False

self._db = db

self._db.registerOnTransactionHandler(self._onTransaction)

self._queue = queue.Queue()

self.serializationContext = TypedPythonCodebase.coreSerializationContext()

self.handler = handler

def setSerializationContext(self, context):

self.serializationContext = context

def start(self):

self._thread.start()

def stop(self):

self._shouldStop = True

self._thread.join()

def __enter__(self):

self.start()

return self

def __exit__(self, exc_type, exc_value, traceback):

self.stop()

def flush(self):

while self._queue.qsize():

time.sleep(0.001)

def _doWork(self):

logger = logging.getLogger(__name__)

while not self._shouldStop:

try:

todo = self._queue.get(timeout=0.1)

except queue.Empty:

todo = None

if todo:

try:

self.handler(todo)

except Exception:

logger.error("Callback threw exception:\n%s", traceback.format_exc())

def _onTransaction(self, key_value, priors, set_adds, set_removes, tid):

changed = {}

for k in key_value:

o, fieldname = self._db._data_key_to_object(k)

if o:

if o not in changed:

changed[o] = []

if fieldname != " exists":

changed[o].append((

fieldname,

View.unwrapSerializedDatabaseValue(self.serializationContext, key_value[k], o.__types__[fieldname]),

View.unwrapSerializedDatabaseValue(self.serializationContext, priors[k], o.__types__[fieldname])

))

self._queue.put(changed)

class DatabaseConnection:

def __init__(self, channel):

self._channel = channel

self._transaction_callbacks = {}

self._lock = threading.Lock()

# transaction of what's in the KV store

self._cur_transaction_num = 0

# a datastructure that keeps track of all the different versions of the objects

# we have mapped in.

self._versioned_data = ManyVersionedObjects()

# a map from lazy object id to (schema, typename)

self._lazy_objects = {}

self._lazy_object_read_blocks = {}

self.initialized = threading.Event()

self.disconnected = threading.Event()

self.connectionObject = None

# transaction handlers. These must be nonblocking since we call them under lock

self._onTransactionHandlers = []

self._flushEvents = {}

# Map: schema.name -> schema

self._schemas = {}

self._messages_received = 0

self._pendingSubscriptions = {}

# if we have object-level subscriptions to a particular type (e.g. not everything)

# then, this is from (schema, typename) -> {object_id -> transaction_id} so that

# we can tell when the subscription should become valid. Subscriptions are permanent

# otherwise, if we're subscribed, it's 'Everything'

self._schema_and_typename_to_subscription_set = {}

# from (schema,typename,field_val) -> {'values', 'index_values', 'identities'}

self._subscription_buildup = {}

self._channel.setServerToClientHandler(self._onMessage)

self._flushIx = 0

self._largeSubscriptionHeartbeatDelay = 0

self.serializationContext = TypedPythonCodebase.coreSerializationContext()

self._logger = logging.getLogger(__name__)

def registerOnTransactionHandler(self, handler):

self._onTransactionHandlers.append(handler)

def setSerializationContext(self, context):

assert isinstance(context, SerializationContext), context

self.serializationContext = context

return self

def serializeFromModule(self, module):

"""Give the project root we want to serialize from."""

self.setSerializationContext(

TypedPythonCodebase.FromRootlevelModule(module).serializationContext

)

def _stopHeartbeating(self):

self._channel._stopHeartbeating()

def disconnect(self):

self.disconnected.set()

self._channel.close()

def _noViewsOutstanding(self):

with self._lock:

return not self._versioned_data._version_number_refcount

def authenticate(self, token):

self._channel.write(

ClientToServer.Authenticate(token=token)

)

def addSchema(self, schema):

schema.freeze()

with self._lock:

if schema.name in self._schemas:

return

self._schemas[schema.name] = schema

schemaDesc = schema.toDefinition()

self._channel.write(

ClientToServer.DefineSchema(

name=schema.name,

definition=schemaDesc

)

)

def flush(self):

"""Make sure we know all transactions that have happened up to this point."""

with self._lock:

if self.disconnected.is_set():

raise DisconnectedException()

self._flushIx += 1

ix = str(self._flushIx)

e = self._flushEvents[ix] = threading.Event()

self._channel.write(ClientToServer.Flush(guid=ix))

e.wait()

if self.disconnected.is_set():

raise DisconnectedException()

def subscribeToObject(self, t):

self.subscribeToObjects([t])

def subscribeToObjects(self, objects):

for t in objects:

self.addSchema(type(t).__schema__)

self.subscribeMultiple([

(type(t).__schema__.name, type(t).__qualname__, ("_identity", t._identity), False)

for t in objects

])

def _lazinessForType(self, typeObj, desiredLaziness):

if desiredLaziness is not None:

return desiredLaziness

if hasattr(typeObj, '__object_database_lazy_subscription__'):

return True

return False

def subscribeToIndex(self, t, block=True, lazySubscription=None, **kwarg):

self.addSchema(t.__schema__)

toSubscribe = []

for fieldname, fieldvalue in kwarg.items():

toSubscribe.append((

t.__schema__.name,

t.__qualname__,

(fieldname, keymapping.index_value_to_hash(fieldvalue)),

self._lazinessForType(t, lazySubscription)

)

)

return self.subscribeMultiple(toSubscribe, block=block)

def subscribeToType(self, t, block=True, lazySubscription=None):

self.addSchema(t.__schema__)

if self._isTypeSubscribedAll(t):

return ()

return self.subscribeMultiple([(t.__schema__.name, t.__qualname__, None, self._lazinessForType(t, lazySubscription))], block)

def subscribeToNone(self, t, block=True):

self.addSchema(t.__schema__)

with self._lock:

self._schema_and_typename_to_subscription_set.setdefault(

(t.__schema__.name, t.__qualname__), set()

)

return ()

def subscribeToSchema(self, *schemas, block=True, lazySubscription=None, excluding=()):

for s in schemas:

self.addSchema(s)

unsubscribedTypes = []

for schema in schemas:

for tname, t in schema._types.items():

if not self._isTypeSubscribedAll(t) and t not in excluding:

unsubscribedTypes.append((schema.name, tname, None, self._lazinessForType(t, lazySubscription)))

if unsubscribedTypes:

return self.subscribeMultiple(unsubscribedTypes, block=block)

return ()

def isSubscribedToSchema(self, schema):

return all(self._isTypeSubscribed(t) for t in schema._types.values())

def isSubscribedToType(self, t):

return self._isTypeSubscribed(t)

def _isTypeSubscribed(self, t):

return (t.__schema__.name, t.__qualname__) in self._schema_and_typename_to_subscription_set

def _isTypeSubscribedAll(self, t):

return self._schema_and_typename_to_subscription_set.get((t.__schema__.name, t.__qualname__)) is Everything

def subscribeMultiple(self, subscriptionTuples, block=True):

with self._lock:

if self.disconnected.is_set():

raise DisconnectedException()

events = []

for tup in subscriptionTuples:

e = self._pendingSubscriptions.get(tup)

if not e:

e = self._pendingSubscriptions[(tup[0], tup[1], tup[2])] = threading.Event()

assert tup[0] and tup[1]

self._channel.write(

ClientToServer.Subscribe(schema=tup[0], typename=tup[1], fieldname_and_value=tup[2], isLazy=tup[3])

)

events.append(e)

if not block:

return tuple(events)

for e in events:

e.wait()

with self._lock:

if self.disconnected.is_set():

raise DisconnectedException()

return ()

def waitForCondition(self, cond, timeout):

# eventally we will replace this with something that watches the calculation

t0 = time.time()

while time.time() - t0 < timeout:

with self.view():

try:

if cond():

return True

except Exception:

self._logger.error("Condition callback threw an exception:\n%s", traceback.format_exc())

time.sleep(min(timeout / 20, .25))

return False

def _data_key_to_object(self, key):

schema_name, typename, identity, fieldname = keymapping.split_data_key(key)

schema = self._schemas.get(schema_name)

if not schema:

return None, None

cls = schema._types.get(typename)

if cls:

return cls.fromIdentity(identity), fieldname

return None, None

def __str__(self):

return "DatabaseConnection(%s)" % id(self)

def __repr__(self):

return "DatabaseConnection(%s)" % id(self)

def current_transaction(self):

if not hasattr(_cur_view, "view"):

return None

return _cur_view.view

def view(self, transaction_id=None):

with self._lock:

if self.disconnected.is_set():

raise DisconnectedException()

if transaction_id is None:

transaction_id = self._cur_transaction_num

assert transaction_id <= self._cur_transaction_num

view = View(self, transaction_id)

self._versioned_data.versionIncref(transaction_id)

return view

def transaction(self):

"""Only one transaction may be committed on the current transaction number."""

with self._lock:

if self.disconnected.is_set():

raise DisconnectedException()

view = Transaction(self, self._cur_transaction_num)

transaction_id = self._cur_transaction_num

self._versioned_data.versionIncref(transaction_id)

return view

def _releaseView(self, view):

with self._lock:

self._versioned_data.versionDecref(view._transaction_num)

def isSubscribedToObject(self, object):

return not self._suppressKey(object._identity)

def _suppressKey(self, k):

schema, typename, ident, fieldname = keymapping.split_data_key(k)

subscriptionSet = self._schema_and_typename_to_subscription_set.get((schema, typename))

if subscriptionSet is Everything:

return False

if isinstance(subscriptionSet, set) and ident in subscriptionSet:

return False

return True

def _suppressIdentities(self, index_key, identities):

schema, typename, fieldname, valhash = keymapping.split_index_key_full(index_key)

subscriptionSet = self._schema_and_typename_to_subscription_set.get((schema, typename))

if subscriptionSet is Everything:

return identities

elif subscriptionSet is None:

return set()

else:

return identities.intersection(subscriptionSet)

def cleanup(self):

with self._lock:

self._versioned_data.cleanup(self._cur_transaction_num)

def _onMessage(self, msg):

self._messages_received += 1

if msg.matches.Disconnected:

with self._lock:

self.disconnected.set()

self.connectionObject = None

for e in self._lazy_object_read_blocks.values():

e.set()

for e in self._flushEvents.values():

e.set()

for e in self._pendingSubscriptions.values():

e.set()

for q in self._transaction_callbacks.values():

try:

q(TransactionResult.Disconnected())

except Exception:

self._logger.error(

"Transaction commit callback threw an exception:\n%s",

traceback.format_exc()

)

self._transaction_callbacks = {}

self._flushEvents = {}

elif msg.matches.FlushResponse:

with self._lock:

e = self._flushEvents.get(msg.guid)

if not e:

self._logger.error("Got an unrequested flush response: %s", msg.guid)

else:

e.set()

elif msg.matches.Initialize:

with self._lock:

self._cur_transaction_num = msg.transaction_num

self.identityProducer = IdentityProducer(msg.identity_root)

self.connectionObject = core_schema.Connection.fromIdentity(msg.connIdentity)

self.initialized.set()

elif msg.matches.TransactionResult:

with self._lock:

try:

self._transaction_callbacks.pop(msg.transaction_guid)(

TransactionResult.Success() if msg.success else

TransactionResult.RevisionConflict(key=msg.badKey)

)

except Exception:

self._logger.error(

"Transaction commit callback threw an exception:\n%s",

traceback.format_exc()

)

elif msg.matches.Transaction:

with self._lock:

key_value = {}

priors = {}

writes = {k: msg.writes[k] for k in msg.writes}

set_adds = {k: msg.set_adds[k] for k in msg.set_adds}

set_removes = {k: msg.set_removes[k] for k in msg.set_removes}

for k, val_serialized in writes.items():

if not self._suppressKey(k):

key_value[k] = val_serialized

priors[k] = self._versioned_data.setVersionedValue(

k, msg.transaction_id,

bytes.fromhex(val_serialized) if val_serialized is not None else None

)

for k, a in set_adds.items():

a = self._suppressIdentities(k, set(a))

self._versioned_data.setVersionedAddsAndRemoves(k, msg.transaction_id, a, set())

for k, r in set_removes.items():

r = self._suppressIdentities(k, set(r))

self._versioned_data.setVersionedAddsAndRemoves(k, msg.transaction_id, set(), r)

self._cur_transaction_num = msg.transaction_id

self._versioned_data.cleanup(self._cur_transaction_num)

for handler in self._onTransactionHandlers:

try:

handler(key_value, priors, set_adds, set_removes, msg.transaction_id)

except Exception:

self._logger.error(

"_onTransaction handler %s threw an exception:\n%s",

handler,

traceback.format_exc()

)

elif msg.matches.SubscriptionIncrease:

with self._lock:

subscribedIdentities = self._schema_and_typename_to_subscription_set.setdefault((msg.schema, msg.typename), set())

if subscribedIdentities is not Everything:

subscribedIdentities.update(

msg.identities

)

elif msg.matches.SubscriptionData:

with self._lock:

lookupTuple = (msg.schema, msg.typename, msg.fieldname_and_value)

if lookupTuple not in self._subscription_buildup:

self._subscription_buildup[lookupTuple] = {'values': {}, 'index_values': {}, 'identities': None, 'markedLazy': False}

else:

assert not self._subscription_buildup[lookupTuple]['markedLazy'], 'received non-lazy data for a lazy subscription'

self._subscription_buildup[lookupTuple]['values'].update({k: msg.values[k] for k in msg.values})

self._subscription_buildup[lookupTuple]['index_values'].update({k: msg.index_values[k] for k in msg.index_values})

if msg.identities is not None:

if self._subscription_buildup[lookupTuple]['identities'] is None:

self._subscription_buildup[lookupTuple]['identities'] = set()

self._subscription_buildup[lookupTuple]['identities'].update(msg.identities)

elif msg.matches.LazyTransactionPriors:

with self._lock:

for k, v in msg.writes.items():

self._versioned_data.setVersionedTailValueStringified(k, bytes.fromhex(v) if v is not None else None)

elif msg.matches.LazyLoadResponse:

with self._lock:

for k, v in msg.values.items():

self._versioned_data.setVersionedTailValueStringified(k, bytes.fromhex(v) if v is not None else None)

self._lazy_objects.pop(msg.identity, None)

e = self._lazy_object_read_blocks.pop(msg.identity, None)

if e:

e.set()

elif msg.matches.LazySubscriptionData:

with self._lock:

lookupTuple = (msg.schema, msg.typename, msg.fieldname_and_value)

assert lookupTuple not in self._subscription_buildup

self._subscription_buildup[lookupTuple] = {

'values': {},

'index_values': msg.index_values,

'identities': msg.identities,

'markedLazy': True

}

elif msg.matches.SubscriptionComplete:

with self._lock:

event = self._pendingSubscriptions.get((

msg.schema,

msg.typename,

tuple(msg.fieldname_and_value) if msg.fieldname_and_value is not None else None

))

if not event:

self._logger.error(

"Received unrequested subscription to schema %s / %s / %s. have %s",

msg.schema, msg.typename, msg.fieldname_and_value, self._pendingSubscriptions

)

return

lookupTuple = (msg.schema, msg.typename, msg.fieldname_and_value)

identities = self._subscription_buildup[lookupTuple]['identities']

values = self._subscription_buildup[lookupTuple]['values']

index_values = self._subscription_buildup[lookupTuple]['index_values']

markedLazy = self._subscription_buildup[lookupTuple]['markedLazy']

del self._subscription_buildup[lookupTuple]

sets = self.indexValuesToSetAdds(index_values)

if msg.fieldname_and_value is None:

if msg.typename is None:

for tname in self._schemas[msg.schema]._types:

self._schema_and_typename_to_subscription_set[msg.schema, tname] = Everything

else:

self._schema_and_typename_to_subscription_set[msg.schema, msg.typename] = Everything

else:

assert msg.typename is not None

subscribedIdentities = self._schema_and_typename_to_subscription_set.setdefault((msg.schema, msg.typename), set())

if subscribedIdentities is not Everything:

subscribedIdentities.update(

identities

)

t0 = time.time()

heartbeatInterval = getHeartbeatInterval()

# this is a fault injection to allow us to verify that heartbeating during this

# function will keep the server connection alive.

for _ in range(self._largeSubscriptionHeartbeatDelay):

self._channel.sendMessage(

ClientToServer.Heartbeat()

)

time.sleep(heartbeatInterval)

totalBytes = 0

for k, v in values.items():

if v is not None:

totalBytes += len(v)

if totalBytes > 1000000:

self._logger.info("Subscription %s loaded %.2f mb of raw data.", lookupTuple, totalBytes / 1024.0 ** 2)

if markedLazy:

schema_and_typename = lookupTuple[:2]

for i in identities:

self._lazy_objects[i] = schema_and_typename

for key, val in values.items():

self._versioned_data.setVersionedValue(key, msg.tid, None if val is None else bytes.fromhex(val))

# this could take a long time, so we need to keep heartbeating

if time.time() - t0 > heartbeatInterval:

# note that this needs to be 'sendMessage' which sends immediately,

# not, 'write' which queues the message after this function finishes!

self._channel.sendMessage(

ClientToServer.Heartbeat()

)

t0 = time.time()

for key, setval in sets.items():

self._versioned_data.updateVersionedAdds(key, msg.tid, set(setval))

# this could take a long time, so we need to keep heartbeating

if time.time() - t0 > heartbeatInterval:

# note that this needs to be 'sendMessage' which sends immediately,

# not, 'write' which queues the message after this function finishes!

self._channel.sendMessage(

ClientToServer.Heartbeat()

)

t0 = time.time()

# this should be inline with the stream of messages coming from the server

assert self._cur_transaction_num <= msg.tid

self._cur_transaction_num = msg.tid

event.set()

else:

assert False, "unknown message type " + msg._which

def indexValuesToSetAdds(self, indexValues):

# indexValues contains (schema:typename:identity:fieldname -> indexHashVal) which builds

# up the indices we need. We need to transpose to a dictionary ordered by the hash values,

# not the identities

t0 = time.time()

heartbeatInterval = getHeartbeatInterval()