omegaml · omegaml · Dec 17, 2024 · Dec 17, 2024
diff --git a/omegaml/mdataframe.py b/omegaml/mdataframe.py
@@ -1,7 +1,5 @@
 from __future__ import absolute_import
 
-import warnings
-
 import numpy as np
 import pandas as pd
 from bson import Code
@@ -200,11 +198,11 @@ def _get_filter(self, specs):
         if isinstance(specs, np.ndarray):
             specs = specs.tolist()
         if (isinstance(specs, enumerable_types)
-              and isscalar(specs[0]) and len(idx_cols) == 1
-              and not any(isinstance(s, slice) for s in specs)):
+                and isscalar(specs[0]) and len(idx_cols) == 1
+                and not any(isinstance(s, slice) for s in specs)):
             # single column index with list of scalar values
             if (self.positional and isinstance(specs, tuple) and len(specs) == 2
-                  and all(isscalar(v) for v in specs)):
+                    and all(isscalar(v) for v in specs)):
                 # iloc[int, int] is a cell access
                 flt_kwargs[idx_cols[0]] = specs[0]
                 projection = self._get_projection(specs[1])
@@ -430,6 +428,7 @@ def _getcopy_kwargs(self, without=None):
                       metadata=self.metadata,
                       query=self.filter_criteria,
                       auto_inspect=self.auto_inspect,
+                      parser=self._parser,
                       preparefn=self._preparefn)
         [kwargs.pop(k) for k in make_tuple(without or [])]
         return kwargs
@@ -767,7 +766,7 @@ def merge(self, right, on=None, left_on=None, right_on=None,
         target_name = self._get_collection_name_of(
             target, '_temp.merge.%s' % uuid4().hex)
         target_field = (
-              "%s_%s" % (right_name.replace('.', '_'), right_on or on))
+                "%s_%s" % (right_name.replace('.', '_'), right_on or on))
         """
         TODO enable filter criteria on right dataframe. requires changing LOOKUP syntax from 
              equitly to arbitray match 
@@ -1010,7 +1009,7 @@ def iterrows(self, chunksize=1000):
             else:
                 # Series does not have iterrows
                 for i in range(0, len(chunkdf), chunksize):
-                    yield chunkdf.iloc[i:i+chunksize]
+                    yield chunkdf.iloc[i:i + chunksize]
 
     def iteritems(self):
         if not hasattr(pd.DataFrame, 'iteritems'):
@@ -1048,7 +1047,6 @@ def rows(self, start=None, end=None, chunksize=1000):
         start, end, chunksize = (int(v) for v in (start, end, chunksize))
         return self.iloc[slice(start, end)].iterchunks(chunksize)
 
-
     def __repr__(self):
         kwargs = ', '.join('{}={}'.format(k, v) for k, v in self._getcopy_kwargs().items())
         return "MDataFrame(collection={collection.name}, {kwargs})".format(collection=self.collection,

diff --git a/omegaml/mixins/store/objinfo.py b/omegaml/mixins/store/objinfo.py
@@ -1,3 +1,4 @@
+from mongoengine import DoesNotExist
 from omegaml.store import OmegaStore
 
 
@@ -18,6 +19,8 @@ def supports(cls, obj, **kwargs):
     def summary(self, name):
         self: OmegaStore | ObjectInformationMixin
         meta = self.metadata(name)
+        if meta is None:
+            raise DoesNotExist(name)
         backend = self.get_backend(name)
         contrib = backend.summary(name) if hasattr(backend, 'summary') else {}
         stats = self._get_collection_stats(meta.collection)
@@ -103,6 +106,8 @@ def _get_database_stats(self, scale=1.0, keys=None):
         try:
             _stats = self.mongodb.command('dbstats', scale=scale)
             _stats['fsAvailableSize'] = _stats['fsTotalSize'] - _stats['fsUsedSize']
+            # prior to 4.4, totalSize was not available
+            _stats.setdefault('totalSize', _stats['dataSize'] + _stats['indexSize'])
         except:
             _stats = {'totalSize': 0, 'fsUsedSize': 0, 'fsTotalSize': 0, 'fsAvailableSize': 0}
         return {

diff --git a/omegaml/notebook/tasks.py b/omegaml/notebook/tasks.py
@@ -6,7 +6,6 @@
 import datetime
 from celery import shared_task
 from celery.utils.log import get_task_logger
-
 from omegaml.celery_util import OmegamlTask, sanitized
 from omegaml.documents import MDREGISTRY
 
@@ -84,12 +83,15 @@ def schedule_omegaml_job(self, nb_file, **kwargs):
 def execute_scripts(self, **kwargs):
     """
     run scheduled jobs
+
+    .. versionchanged:: 0.16.4
+       hidden jobs are now included in the list of jobs to run
     """
     logger = get_task_logger(self.name)
     om = self.om
     now = kwargs.get('now') or datetime.datetime.now()
     # get pending tasks, execute if time is right
-    for job_meta in om.jobs.list(raw=True):
+    for job_meta in om.jobs.list(raw=True, hidden=True):
         if job_meta.name.startswith('results'):
             # ignore any scheduled results
             continue

diff --git a/omegaml/runtimes/mixins/swagger/helpers.py b/omegaml/runtimes/mixins/swagger/helpers.py
@@ -220,13 +220,13 @@ def datatypes(self, orient):
                                                many=spec.get('many', False))
                 for k, spec in signature.items()
                 if (k in ('X', 'Y', 'result')
-                    and (spec or {}).get('schema')) # skip empty schemas
+                    and (spec or {}).get('schema'))  # skip empty schemas
             }
             # errors - multiple response types, each mapped to one http status
             errors = {
                 int(status): store._datatype_from_schema(spec['schema'], name=f'{name}_{status}',
-                                               orient=orient,
-                                               many=spec.get('many', False))
+                                                         orient=orient,
+                                                         many=spec.get('many', False))
                 for status, spec in signature.get('errors').items()
             }
             # include a standard error message
@@ -238,7 +238,8 @@ def datatypes(self, orient):
         else:
             datatypes = None
         if not datatypes:
-            raise ValueError(f"Missing signature specification for object {store.prefix}/{name}")
+            resource = f'{store.prefix}/{name}'.replace('//', '/')
+            raise ValueError(f"Missing signature specification for object {resource}")
         return datatypes
 
 

diff --git a/omegaml/runtimes/proxies/modelproxy.py b/omegaml/runtimes/proxies/modelproxy.py
@@ -110,3 +110,11 @@ def experiments(self, label=None, raw=False):
         }
         all_exps = dict(**by_label, **unlabeled)
         return {k: v for k, v in all_exps.items() if not label or k == label}
+
+    def monitor(self, label=None, provider=None, **tracker_kwargs):
+        """ return the monitor for this model
+
+        Returns:
+            OmegaMonitorProxy() instance
+        """
+        return self.experiment(label=label, provider=provider, **tracker_kwargs).as_monitor(self.modelname)
diff --git a/omegaml/store/base.py b/omegaml/store/base.py
@@ -703,7 +703,7 @@ def drop(self, name, force=False, version=-1, report=False, **kwargs):
     def _drop(self, name, force=False, version=-1, **kwargs):
         meta = self.metadata(name, version=version)
         if meta is None and not force:
-            raise DoesNotExist()
+            raise DoesNotExist(name)
         collection = self.collection(name)
         if collection:
             self.mongodb.drop_collection(collection.name)

diff --git a/omegaml/store/queryops.py b/omegaml/store/queryops.py
@@ -5,6 +5,7 @@
 import sys
 import warnings
 from hashlib import md5
+
 from omegaml.util import make_tuple, is_interactive, signature
 
 
@@ -56,7 +57,7 @@ def get_coordinates_from_geojson(self, d):
             coordinates = d.get('geometry').get('coordinates')
         else:
             raise ValueError(
-                'expected a valid GeoJSON dict, got %s' % coordinates)
+                'expected a valid GeoJSON dict, got %s' % d)
         return coordinates
 
     @property

diff --git a/omegaml/tests/util.py b/omegaml/tests/util.py
@@ -2,31 +2,45 @@
 
 import os
 import warnings
+
 from omegaml import Omega
+from omegaml.client.lunamon import LunaMonitor
 
 
 class OmegaTestMixin(object):
     def setUp(self):
+        super().setUp()
         self.om = Omega()
         self.clean()
 
+    def tearDown(self):
+        super().tearDown()
+        LunaMonitor.stop_all()
+
     def shortDescription(self):
         # always print method name instead of docstring
         # see unittest.TestCase for details
         return None
 
     def clean(self, bucket=None):
+        # stop monitoring to avoid interference with tests
+        LunaMonitor.stop_all()
+        # clean om stores
         om = self.om[bucket] if bucket is not None else self.om
         for element in ('models', 'jobs', 'datasets', 'scripts', 'streams'):
             part = getattr(om, element)
             drop = part.drop
             drop_kwargs = {}
             if element == 'streams':
                 drop_kwargs = {'keep_data': False}
+            # drop all members
             [drop(m.name,
                   force=True,
                   **drop_kwargs) for m in part.list(hidden=True, include_temp=True, raw=True)]
-            self.assertListEqual(part.list(hidden=True, include_temp=True), [])
+            # ignore system members, as they may get recreated e.g. by LunaMonitor
+            existing = [m.name for m in part.list(hidden=True, include_temp=True)
+                        if not m.name.startswith('.system')]
+            self.assertListEqual(existing, [])
 
     @property
     def _async_headers(self):

diff --git a/omegaml/util.py b/omegaml/util.py
@@ -1303,13 +1303,18 @@ def inprogress(text="running {fn}", **__kwargs):
     # print a message when entering a function
     # -- useful for debugging
     # -- use as a decorator
-    if not module_available('yaspin'):
-        @contextmanager
-        def yaspin(*args, text=None, **kwargs):
-            logger.debug(text)
-            yield
-    else:
-        from yaspin import yaspin
+    def is_piped():
+        return not sys.stdout.isatty()
+
+    def is_running_in_jupyter():
+        try:
+            from IPython import get_ipython
+            return get_ipython() is not None
+        except ImportError:
+            return False
+
+    should_spin = (is_running_in_jupyter() or not is_piped())
+    yaspin = failsafe_yaspin(mock=not should_spin)
 
     def decorator(fn):
         def wrapper(*args, **kwargs):
@@ -1340,3 +1345,16 @@ def ensurelist(l):
     return l.tolist() if isinstance(l, np.ndarray) else list(l)
 
 
+def failsafe_yaspin(mock=False):
+    try:
+        if not mock:
+            from yaspin import yaspin
+        else:
+            raise ImportError('not loading yaspin due to mock=True')
+    except Exception as e:
+        @contextmanager
+        def yaspin(*args, text=None, **kwargs):
+            setattr(yaspin, 'text', text)
+            logger.debug(getattr(yaspin, 'text', '...'))
+            yield
+    return yaspin