Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit dca0ae1

Browse files
omegamlomegaml
authored
Merge pull request #462 from omegaml/various
chore: various updates
2 parents 95f2a45 + 15f2503 commit dca0ae1

File tree

9 files changed

+71
-24
lines changed

9 files changed

+71
-24
lines changed

‎omegaml/mdataframe.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
from __future__ import absolute_import
22

3-
import warnings
4-
53
import numpy as np
64
import pandas as pd
75
from bson import Code
@@ -200,11 +198,11 @@ def _get_filter(self, specs):
200198
if isinstance(specs, np.ndarray):
201199
specs = specs.tolist()
202200
if (isinstance(specs, enumerable_types)
203-
and isscalar(specs[0]) and len(idx_cols) == 1
204-
and not any(isinstance(s, slice) for s in specs)):
201+
and isscalar(specs[0]) and len(idx_cols) == 1
202+
and not any(isinstance(s, slice) for s in specs)):
205203
# single column index with list of scalar values
206204
if (self.positional and isinstance(specs, tuple) and len(specs) == 2
207-
and all(isscalar(v) for v in specs)):
205+
and all(isscalar(v) for v in specs)):
208206
# iloc[int, int] is a cell access
209207
flt_kwargs[idx_cols[0]] = specs[0]
210208
projection = self._get_projection(specs[1])
@@ -430,6 +428,7 @@ def _getcopy_kwargs(self, without=None):
430428
metadata=self.metadata,
431429
query=self.filter_criteria,
432430
auto_inspect=self.auto_inspect,
431+
parser=self._parser,
433432
preparefn=self._preparefn)
434433
[kwargs.pop(k) for k in make_tuple(without or [])]
435434
return kwargs
@@ -767,7 +766,7 @@ def merge(self, right, on=None, left_on=None, right_on=None,
767766
target_name = self._get_collection_name_of(
768767
target, '_temp.merge.%s' % uuid4().hex)
769768
target_field = (
770-
"%s_%s" % (right_name.replace('.', '_'), right_on or on))
769+
"%s_%s" % (right_name.replace('.', '_'), right_on or on))
771770
"""
772771
TODO enable filter criteria on right dataframe. requires changing LOOKUP syntax from
773772
equitly to arbitray match
@@ -1010,7 +1009,7 @@ def iterrows(self, chunksize=1000):
10101009
else:
10111010
# Series does not have iterrows
10121011
for i in range(0, len(chunkdf), chunksize):
1013-
yield chunkdf.iloc[i:i+chunksize]
1012+
yield chunkdf.iloc[i:i + chunksize]
10141013

10151014
def iteritems(self):
10161015
if not hasattr(pd.DataFrame, 'iteritems'):
@@ -1048,7 +1047,6 @@ def rows(self, start=None, end=None, chunksize=1000):
10481047
start, end, chunksize = (int(v) for v in (start, end, chunksize))
10491048
return self.iloc[slice(start, end)].iterchunks(chunksize)
10501049

1051-
10521050
def __repr__(self):
10531051
kwargs = ', '.join('{}={}'.format(k, v) for k, v in self._getcopy_kwargs().items())
10541052
return "MDataFrame(collection={collection.name}, {kwargs})".format(collection=self.collection,

‎omegaml/mixins/store/objinfo.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from mongoengine import DoesNotExist
12
from omegaml.store import OmegaStore
23

34

@@ -18,6 +19,8 @@ def supports(cls, obj, **kwargs):
1819
def summary(self, name):
1920
self: OmegaStore | ObjectInformationMixin
2021
meta = self.metadata(name)
22+
if meta is None:
23+
raise DoesNotExist(name)
2124
backend = self.get_backend(name)
2225
contrib = backend.summary(name) if hasattr(backend, 'summary') else {}
2326
stats = self._get_collection_stats(meta.collection)
@@ -103,6 +106,8 @@ def _get_database_stats(self, scale=1.0, keys=None):
103106
try:
104107
_stats = self.mongodb.command('dbstats', scale=scale)
105108
_stats['fsAvailableSize'] = _stats['fsTotalSize'] - _stats['fsUsedSize']
109+
# prior to 4.4, totalSize was not available
110+
_stats.setdefault('totalSize', _stats['dataSize'] + _stats['indexSize'])
106111
except:
107112
_stats = {'totalSize': 0, 'fsUsedSize': 0, 'fsTotalSize': 0, 'fsAvailableSize': 0}
108113
return {

‎omegaml/notebook/tasks.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import datetime
77
from celery import shared_task
88
from celery.utils.log import get_task_logger
9-
109
from omegaml.celery_util import OmegamlTask, sanitized
1110
from omegaml.documents import MDREGISTRY
1211

@@ -84,12 +83,15 @@ def schedule_omegaml_job(self, nb_file, **kwargs):
8483
def execute_scripts(self, **kwargs):
8584
"""
8685
run scheduled jobs
86+
87+
.. versionchanged:: 0.16.4
88+
hidden jobs are now included in the list of jobs to run
8789
"""
8890
logger = get_task_logger(self.name)
8991
om = self.om
9092
now = kwargs.get('now') or datetime.datetime.now()
9193
# get pending tasks, execute if time is right
92-
for job_meta in om.jobs.list(raw=True):
94+
for job_meta in om.jobs.list(raw=True, hidden=True):
9395
if job_meta.name.startswith('results'):
9496
# ignore any scheduled results
9597
continue

‎omegaml/runtimes/mixins/swagger/helpers.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -220,13 +220,13 @@ def datatypes(self, orient):
220220
many=spec.get('many', False))
221221
for k, spec in signature.items()
222222
if (k in ('X', 'Y', 'result')
223-
and (spec or {}).get('schema')) # skip empty schemas
223+
and (spec or {}).get('schema')) # skip empty schemas
224224
}
225225
# errors - multiple response types, each mapped to one http status
226226
errors = {
227227
int(status): store._datatype_from_schema(spec['schema'], name=f'{name}_{status}',
228-
orient=orient,
229-
many=spec.get('many', False))
228+
orient=orient,
229+
many=spec.get('many', False))
230230
for status, spec in signature.get('errors').items()
231231
}
232232
# include a standard error message
@@ -238,7 +238,8 @@ def datatypes(self, orient):
238238
else:
239239
datatypes = None
240240
if not datatypes:
241-
raise ValueError(f"Missing signature specification for object {store.prefix}/{name}")
241+
resource = f'{store.prefix}/{name}'.replace('//', '/')
242+
raise ValueError(f"Missing signature specification for object {resource}")
242243
return datatypes
243244

244245

‎omegaml/runtimes/proxies/modelproxy.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,11 @@ def experiments(self, label=None, raw=False):
110110
}
111111
all_exps = dict(**by_label, **unlabeled)
112112
return {k: v for k, v in all_exps.items() if not label or k == label}
113+
114+
def monitor(self, label=None, provider=None, **tracker_kwargs):
115+
""" return the monitor for this model
116+
117+
Returns:
118+
OmegaMonitorProxy() instance
119+
"""
120+
return self.experiment(label=label, provider=provider, **tracker_kwargs).as_monitor(self.modelname)

‎omegaml/store/base.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -703,7 +703,7 @@ def drop(self, name, force=False, version=-1, report=False, **kwargs):
703703
def _drop(self, name, force=False, version=-1, **kwargs):
704704
meta = self.metadata(name, version=version)
705705
if meta is None and not force:
706-
raise DoesNotExist()
706+
raise DoesNotExist(name)
707707
collection = self.collection(name)
708708
if collection:
709709
self.mongodb.drop_collection(collection.name)

‎omegaml/store/queryops.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import sys
66
import warnings
77
from hashlib import md5
8+
89
from omegaml.util import make_tuple, is_interactive, signature
910

1011

@@ -56,7 +57,7 @@ def get_coordinates_from_geojson(self, d):
5657
coordinates = d.get('geometry').get('coordinates')
5758
else:
5859
raise ValueError(
59-
'expected a valid GeoJSON dict, got %s' % coordinates)
60+
'expected a valid GeoJSON dict, got %s' % d)
6061
return coordinates
6162

6263
@property

‎omegaml/tests/util.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,31 +2,45 @@
22

33
import os
44
import warnings
5+
56
from omegaml import Omega
7+
from omegaml.client.lunamon import LunaMonitor
68

79

810
class OmegaTestMixin(object):
911
def setUp(self):
12+
super().setUp()
1013
self.om = Omega()
1114
self.clean()
1215

16+
def tearDown(self):
17+
super().tearDown()
18+
LunaMonitor.stop_all()
19+
1320
def shortDescription(self):
1421
# always print method name instead of docstring
1522
# see unittest.TestCase for details
1623
return None
1724

1825
def clean(self, bucket=None):
26+
# stop monitoring to avoid interference with tests
27+
LunaMonitor.stop_all()
28+
# clean om stores
1929
om = self.om[bucket] if bucket is not None else self.om
2030
for element in ('models', 'jobs', 'datasets', 'scripts', 'streams'):
2131
part = getattr(om, element)
2232
drop = part.drop
2333
drop_kwargs = {}
2434
if element == 'streams':
2535
drop_kwargs = {'keep_data': False}
36+
# drop all members
2637
[drop(m.name,
2738
force=True,
2839
**drop_kwargs) for m in part.list(hidden=True, include_temp=True, raw=True)]
29-
self.assertListEqual(part.list(hidden=True, include_temp=True), [])
40+
# ignore system members, as they may get recreated e.g. by LunaMonitor
41+
existing = [m.name for m in part.list(hidden=True, include_temp=True)
42+
if not m.name.startswith('.system')]
43+
self.assertListEqual(existing, [])
3044

3145
@property
3246
def _async_headers(self):

‎omegaml/util.py‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1303,13 +1303,18 @@ def inprogress(text="running {fn}", **__kwargs):
13031303
# print a message when entering a function
13041304
# -- useful for debugging
13051305
# -- use as a decorator
1306-
if not module_available('yaspin'):
1307-
@contextmanager
1308-
def yaspin(*args, text=None, **kwargs):
1309-
logger.debug(text)
1310-
yield
1311-
else:
1312-
from yaspin import yaspin
1306+
def is_piped():
1307+
return not sys.stdout.isatty()
1308+
1309+
def is_running_in_jupyter():
1310+
try:
1311+
from IPython import get_ipython
1312+
return get_ipython() is not None
1313+
except ImportError:
1314+
return False
1315+
1316+
should_spin = (is_running_in_jupyter() or not is_piped())
1317+
yaspin = failsafe_yaspin(mock=not should_spin)
13131318

13141319
def decorator(fn):
13151320
def wrapper(*args, **kwargs):
@@ -1340,3 +1345,16 @@ def ensurelist(l):
13401345
return l.tolist() if isinstance(l, np.ndarray) else list(l)
13411346

13421347

1348+
def failsafe_yaspin(mock=False):
1349+
try:
1350+
if not mock:
1351+
from yaspin import yaspin
1352+
else:
1353+
raise ImportError('not loading yaspin due to mock=True')
1354+
except Exception as e:
1355+
@contextmanager
1356+
def yaspin(*args, text=None, **kwargs):
1357+
setattr(yaspin, 'text', text)
1358+
logger.debug(getattr(yaspin, 'text', '...'))
1359+
yield
1360+
return yaspin

0 commit comments

Comments
 (0)