Merge pull request #377 from omegaml/release-build-0.16.2

omegaml · web-flow · commit 69603efba871 · 2024-03-08T14:41:55.000+01:00
Release build 0.16.2
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.16.1
+current_version = 0.16.2
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)([-](?P<release>(dev|rc))+(?P<build>\d+))?
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -119,6 +119,7 @@ jobs:
             chmod -R 777 /tmp/testlogs
             # build python specific wheel to allow dependency matching, e.g. tensorflow 2.5 requires py3.9
             # TODO consider using https://cibuildwheel.readthedocs.io or https://github.com/getsentry/milksnake or tox
+            # PYTHON 3.12 is pending on https://github.com/jupyter/docker-stacks/pull/2072
             for PYVER in 3.10 3.11; do
               echo "Building distribution for $PYVER"
               # setup pyenv and build
diff --git a/Makefile b/Makefile
@@ -53,7 +53,8 @@ release-test: bumpbuild dist
 	: "twine upload to pypi test"
 	# see https://packaging.python.org/tutorials/packaging-projects/
 	# config is in $HOME/.pypirc
-	twine upload --repository testpypi-omegaml dist/*gz dist/*whl
+	twine upload --skip-existing --repository testpypi-omegaml dist/*gz
+	twine upload --repository testpypi-omegaml dist/*whl
 	sleep 5
 	scripts/livetest.sh --testpypi --build
 
@@ -106,7 +107,7 @@ bumpminor:
 	bumpversion minor
 
 bumpbuild:
-	bumpversion build
+	[ ! -z "${CI_PULL_REQUEST}" ] && echo "CI_PULL_REQUEST is set, not bumping" || bumpversion build
 
 bumpfinal:
 	bumpversion release
@@ -129,3 +130,4 @@ pipsync:
 scan: freeze pipsync
 	snyk test --policy-path=./.snyk
 	snyk code test
+	mv requirements.txt scanned-pipreqs.txt
diff --git a/omegaml/VERSION b/omegaml/VERSION
@@ -1,2 +1,2 @@
-0.16.1
+0.16.2
 
diff --git a/omegaml/backends/virtualobj.py b/omegaml/backends/virtualobj.py
@@ -5,6 +5,7 @@
 import warnings
 
 from omegaml.backends.basedata import BaseDataBackend
+from omegaml.util import tryOr
 
 
 class VirtualObjectBackend(BaseDataBackend):
@@ -255,7 +256,7 @@ def loads(self, data):
             # this is to deal with functions created outside of __main__
             # see https://stackoverflow.com/q/26193102/890242
             #     https://stackoverflow.com/a/70513630/890242
-            mod = types.ModuleType(e.name, 'dynamic module')
+            mod = types.ModuleType(e.name, '__dynamic__')
             sys.modules[e.name] = mod  # sys.modules['__main__']
             obj = dill.loads(data)
         return obj
@@ -348,7 +349,7 @@ def _dynamic_compile(self, obj, module='__main__'):
         return obj
 
     def isdipped(self, data_or_obj):
-        obj = dill.loads(data_or_obj) if not isinstance(data_or_obj, dict) else data_or_obj
+        obj = tryOr(lambda : dill.loads(data_or_obj), None) if not isinstance(data_or_obj, dict) else data_or_obj
         return isinstance(obj, dict) and obj.get('__dipped__') == self.__calories
 
 
diff --git a/omegaml/client/cli/runtime.py b/omegaml/client/cli/runtime.py
@@ -25,7 +25,7 @@ class RuntimeCommandBase(CommandBase):
       om runtime log [-f] [options]
       om runtime status [workers|labels|stats] [options]
       om runtime restart app <name> [--insecure] [--apphub-url=<url>] [options]
-      om runtime serve [<rule>...] [--rules=<rulefile>] [options]
+      om runtime serve [<rule>...] [--rules=<rulefile>] [--port=<port>] [--ip=<ip>] [options]
       om runtime deploy [<deploy-action>] [--steps=<deployfile>] [--specs=<specs>] [--select=<filter>] [--dry] [options]
       om runtime (control|inspect|celery) [<celery-command>...] [--worker=<worker>] [--queue=<queue>] [--celery-help] [--flags <celery-flags>] [options]
       om runtime (export|import) [<prefix/name>...] [--path=<path>] [--compress] [--list] [--promote] [options]
@@ -45,6 +45,8 @@ class RuntimeCommandBase(CommandBase):
       --rules=VALUE       /path/to/specs.txt, where each line is a <spec>
       --compress          if specified the archive will be compress (tgz format) [default: True]
       --path=PATH         path to directory where the archive should be written [default: ./mlops-export]
+      --port=VALUE        the port to use for the server [default: 8000]
+      --ip=VALUE          the ip to use for the server [default: localhost]
       --list              if specified, print members of archive
       --promote           if specified, import and promote objects
       --steps=VALUE       /path/to/deployfile.yaml [default: ./deployfile.yaml]
@@ -429,11 +431,13 @@ def serve(self):
         om = get_omega(self.args, require_config=False)
         specs = self.args.get('<rule>')
         specfile = self.args.get('--rules')
+        port = self.args.get('--port', 8000)
+        host = self.args.get('--host', 'localhost')
         if specfile:
             with open(specfile, 'r') as fin:
                 specs = [s.replace('\n', '') for s in fin.readlines() if not s.startswith('#')]
         os.environ['OMEGA_RESTAPI_FILTER'] = ';'.join(specs) if specs else om.defaults.OMEGA_RESTAPI_FILTER
-        subprocess.run("gunicorn 'omegaml.restapi.app:serve_objects()'", shell=True)
+        subprocess.run(f"gunicorn 'omegaml.restapi.app:serve_objects()' --bind {host}:{port}", shell=True)
 
     def deploy(self):
         om = get_omega(self.args, require_config=False)
diff --git a/omegaml/defaults.py b/omegaml/defaults.py
@@ -121,7 +121,7 @@
     'model.r': 'omegaml.backends.rsystem.rmodels.RModelBackend',
     'package.r': 'omegaml.backends.rsystem.rscripts.RPackageData',
 }
-#: supported frameworks
+#: supported frameworks (deprecated since 0.16.2, it is effectively ignored)
 OMEGA_FRAMEWORKS = os.environ.get('OMEGA_FRAMEWORKS', 'scikit-learn').split(',')
 if is_test_run:
     OMEGA_FRAMEWORKS = ('scikit-learn', 'tensorflow', 'keras')
@@ -394,14 +394,14 @@ def load_framework_support(vars=globals()):
     # -- note we do this here to ensure this happens after config updates
     if OMEGA_DISABLE_FRAMEWORKS:
         return
-    if 'tensorflow' in vars['OMEGA_FRAMEWORKS'] and tensorflow_available():
+    if tensorflow_available():
         #: tensorflow backend
         # https://stackoverflow.com/a/38645250
         os.environ['TF_CPP_MIN_LOG_LEVEL'] = os.environ.get('TF_CPP_MIN_LOG_LEVEL') or '3'
         logging.getLogger('tensorflow').setLevel(logging.ERROR)
         vars['OMEGA_STORE_BACKENDS'].update(vars['OMEGA_STORE_BACKENDS_TENSORFLOW'])
     #: keras backend
-    if 'keras' in vars['OMEGA_FRAMEWORKS'] and keras_available():
+    if keras_available():
         vars['OMEGA_STORE_BACKENDS'].update(vars['OMEGA_STORE_BACKENDS_KERAS'])
     #: sqlalchemy backend
     if module_available('sqlalchemy'):
diff --git a/omegaml/util.py b/omegaml/util.py
@@ -644,6 +644,7 @@ def module_available(modname):
 
 
 def tensorflow_available():
+    os.environ.setdefault('TF_CPP_MIN_LOG_LEVEL', '3')
     return module_available('tensorflow')
 
 
@@ -1088,7 +1089,7 @@ class SystemPosixPath(type(Path()), Path):
         => str(path) => will always be './foo/bar')
 
     Rationale:
-        pathlib.Path('./foo/bar') will render as '\\foo\bar' on Windows
+        pathlib.Path('./foo/bar') will render as '\\foo\\bar' on Windows
         while pathlib.PosixPath cannot be imported in Windows
     """
 
@@ -1166,9 +1167,9 @@ def tarfile_safe_extractall(tar, dest_path, filter='data'):
             # Path is absolute even after stripping.
             # For example, 'C:/foo' on Windows.
             raise ValueError(f"tarfile: {member} is an absolute path")
-            # Ensure we stay in the destination
-        target_path = os.path.realpath(os.path.join(dest_path, name))
-        if os.path.commonpath([target_path, dest_path]) != dest_path:
+        # Ensure we stay in the destination
+        target_path = (Path(dest_path) / name).resolve()
+        if not target_path.is_relative_to(dest_path):
             raise ValueError(f"tarfile: {member}, {target_path} is outside of destination")
     # Python before 3.12, before backports to 3.11, 3.10, 3.9
     return tar.extractall(dest_path)
diff --git a/scanned-pipreqs.txt b/scanned-pipreqs.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --output-file=requirements.txt
@@ -77,6 +77,8 @@ dnspython==2.5.0
     #   pymongo
 docopt==0.6.2
     # via omegaml (setup.py)
+exceptiongroup==1.2.0
+    # via ipython
 executing==2.0.1
     # via stack-data
 fastjsonschema==2.19.1
