From 9d55f618b31e8c9aa467d67b3e94f6abc53859e0 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 21 Aug 2018 16:55:25 +0200 Subject: [PATCH 1/6] [scripts] Add odml to RDF export --- odml/scripts/odml_export_rdf.py | 152 ++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 odml/scripts/odml_export_rdf.py diff --git a/odml/scripts/odml_export_rdf.py b/odml/scripts/odml_export_rdf.py new file mode 100644 index 00000000..0d53f0e5 --- /dev/null +++ b/odml/scripts/odml_export_rdf.py @@ -0,0 +1,152 @@ +"""odmlExportRDF + +odmlExportRDF searches for odML files within a provided SEARCHDIR +and converts them to the newest odML format version and +exports all found and resulting odML files to XML formatted RDF. +Original files will never be overwritten. + +Usage: odmlexportrdf [-r] [-o OUT] SEARCHDIR + +Arguments: + SEARCHDIR Directory to search for odML files. + +Options: + -o OUT Output directory. Must exist if specified. + If not specified, output files will be + written to the current directory. + -r Search recursively. + -h --help Show this screen. + --version Show version. +""" + +import os +import pathlib +import sys +import tempfile + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +import odml + +from docopt import docopt +from odml.tools.odmlparser import ODMLReader, ODMLWriter +from odml.tools.version_converter import VersionConverter as VerConf + +try: + unicode = unicode +except NameError: + unicode = str + + +def run_rdf_export(odml_file, export_dir): + """ + Convert an odML file to an XML RDF file and + export it to an export directory with the + same name as the original file and a '.rdf' file + ending. + :param odml_file: odML file to be converted to RDF. + :param export_dir: + """ + out_name = os.path.splitext(os.path.basename(odml_file))[0] + out_file = os.path.join(export_dir, "%s.rdf" % out_name) + doc = ODMLReader().from_file(odml_file) + ODMLWriter("RDF").write_file(doc, out_file) + + +def run_conversion(file_list, output_dir, rdf_dir, report, source_format="XML"): + """ + Convert a list of odML files to the latest odML version if required + and export all files to XML RDF files in a specified output directory. + :param file_list: list of files to be exported to RDF. + :param output_dir: Directory where odML files converted to + the latest odML version will be saved. + :param rdf_dir: Directory where exported RDF files will be saved. + :param report: Reporting StringIO. + :param source_format: Original file format of the odML source files. + XML, JSON and YAML are supported, default is XML. + """ + # Exceptions are kept as broad as possible to ignore any non-odML or + # invalid odML files and ensuring everything that can be will be converted. + for curr_file in file_list: + file_path = unicode(curr_file.absolute()) + report.write("[Info] Handling file '%s'\n" % file_path) + # When loading the current file succeeds, it is + # a recent odML format file and can be exported + # to RDF right away. Otherwise it needs to be + # converted to the latest odML version first. + try: + odml.load(file_path, source_format) + report.write("[Info] RDF conversion of '%s'\n" % file_path) + run_rdf_export(file_path, rdf_dir) + except Exception as exc: + out_name = os.path.splitext(os.path.basename(file_path))[0] + outfile = os.path.join(output_dir, "%s_conv.xml" % out_name) + try: + VerConf(file_path).write_to_file(outfile, source_format) + try: + report.write("[Info] RDF conversion of '%s'\n" % outfile) + run_rdf_export(outfile, rdf_dir) + except Exception as exc: + report.write("[Error] converting '%s' to RDF: '%s'\n" % + (file_path, exc)) + except Exception as exc: + # Ignore files we cannot parse or convert + report.write("[Error] version converting file '%s': '%s'\n" % + (file_path, exc)) + + +def main(args=None): + """ + Convenience script to automatically convert odML files + within a directory (tree) to RDF. Check the cli help + for details. + :param args: Command line arguments + """ + parser = docopt(__doc__, argv=args, version="0.1.0") + + root = parser['SEARCHDIR'] + if not os.path.isdir(root): + print(docopt(__doc__, "-h")) + exit(1) + + # Handle all supported odML file formats. + if parser['-r']: + xfiles = list(pathlib.Path(root).rglob('*.odml')) + xfiles.extend(list(pathlib.Path(root).rglob('*.xml'))) + jfiles = list(pathlib.Path(root).rglob('*.json')) + yfiles = list(pathlib.Path(root).rglob('*.yaml')) + else: + xfiles = list(pathlib.Path(root).glob('*.odml')) + xfiles.extend(list(pathlib.Path(root).glob('*.xml'))) + jfiles = list(pathlib.Path(root).glob('*.json')) + yfiles = list(pathlib.Path(root).glob('*.yaml')) + + out_root = os.getcwd() + if parser["-o"]: + if not os.path.isdir(parser["-o"]): + print("[Error] Could not find output directory '%s'" % parser["-o"]) + exit(1) + + out_root = parser["-o"] + + out_dir = tempfile.mkdtemp(prefix="odmlconv_", dir=out_root) + rdf_dir = tempfile.mkdtemp(prefix="odmlrdf_", dir=out_dir) + + # Use this monkeypatch reporter until there is a way + # to run the converters silently. + report = StringIO() + report.write("[Info] Files will be saved to '%s'\n" % out_dir) + + run_conversion(xfiles, out_dir, rdf_dir, report) + run_conversion(jfiles, out_dir, rdf_dir, report, "JSON") + run_conversion(yfiles, out_dir, rdf_dir, report, "YAML") + + print(report.getvalue()) + report.close() + + +if __name__ == "__main__": + main(sys.argv[1:]) From d4f46f9cc41247a78d910e1b7e61ae7922fea199 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 21 Aug 2018 17:23:26 +0200 Subject: [PATCH 2/6] [odml/scripts] Add init file for console scripts --- odml/scripts/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 odml/scripts/__init__.py diff --git a/odml/scripts/__init__.py b/odml/scripts/__init__.py new file mode 100644 index 00000000..e69de29b From 6838e1f83ee027b53e9c1ab3952331156ce92269 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 21 Aug 2018 17:23:48 +0200 Subject: [PATCH 3/6] [setup] Add console scripts --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index f0e5d053..9208b298 100644 --- a/setup.py +++ b/setup.py @@ -21,13 +21,14 @@ packages = [ 'odml', - 'odml.tools' + 'odml.tools', + 'odml.scripts' ] with open('README.rst') as f: description_text = f.read() -install_req = ["lxml", "pyyaml==3.12", "rdflib"] +install_req = ["lxml", "pyyaml==3.12", "rdflib", "docopt"] if sys.version_info < (3, 4): install_req += ["enum34"] @@ -45,5 +46,6 @@ include_package_data=True, long_description=description_text, classifiers=CLASSIFIERS, - license="BSD" + license="BSD", + entry_points={'console_scripts': ['odmlexportrdf=odml.scripts.odml_export_rdf:main']} ) From 5d0ae83a84e61ae8bb10e28cf56515d3b56d0a43 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Tue, 21 Aug 2018 17:24:07 +0200 Subject: [PATCH 4/6] [info] Update version to 1.4.1 --- odml/info.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odml/info.json b/odml/info.json index b4a26470..4ef36c13 100644 --- a/odml/info.json +++ b/odml/info.json @@ -1,5 +1,5 @@ { - "VERSION": "1.4.0", + "VERSION": "1.4.1", "FORMAT_VERSION": "1.1", "AUTHOR": "Hagen Fritsch, Jan Grewe, Christian Kellner, Achilleas Koutsou, Michael Sonntag, Lyuba Zehl", "COPYRIGHT": "(c) 2011-2018, German Neuroinformatics Node", From 6d84a43185ad40d19369060caec7dba91c728a34 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 22 Aug 2018 10:04:00 +0200 Subject: [PATCH 5/6] [odml/scripts] Rename to odmltordf --- odml/scripts/{odml_export_rdf.py => odml_to_rdf.py} | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename odml/scripts/{odml_export_rdf.py => odml_to_rdf.py} (97%) diff --git a/odml/scripts/odml_export_rdf.py b/odml/scripts/odml_to_rdf.py similarity index 97% rename from odml/scripts/odml_export_rdf.py rename to odml/scripts/odml_to_rdf.py index 0d53f0e5..61bc53b6 100644 --- a/odml/scripts/odml_export_rdf.py +++ b/odml/scripts/odml_to_rdf.py @@ -1,11 +1,11 @@ -"""odmlExportRDF +"""odmlToRDF -odmlExportRDF searches for odML files within a provided SEARCHDIR +odmlToRDF searches for odML files within a provided SEARCHDIR and converts them to the newest odML format version and exports all found and resulting odML files to XML formatted RDF. Original files will never be overwritten. -Usage: odmlexportrdf [-r] [-o OUT] SEARCHDIR +Usage: odmltordf [-r] [-o OUT] SEARCHDIR Arguments: SEARCHDIR Directory to search for odML files. diff --git a/setup.py b/setup.py index 9208b298..e67a134c 100644 --- a/setup.py +++ b/setup.py @@ -47,5 +47,5 @@ long_description=description_text, classifiers=CLASSIFIERS, license="BSD", - entry_points={'console_scripts': ['odmlexportrdf=odml.scripts.odml_export_rdf:main']} + entry_points={'console_scripts': ['odmltordf=odml.scripts.odml_to_rdf:main']} ) From a4a2c6705cb18b055a78510ea673a34c6c4d30b0 Mon Sep 17 00:00:00 2001 From: "M. Sonntag" Date: Wed, 22 Aug 2018 10:07:20 +0200 Subject: [PATCH 6/6] [setup] Upgrade to pyyaml 3.13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e67a134c..bade83c1 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ with open('README.rst') as f: description_text = f.read() -install_req = ["lxml", "pyyaml==3.12", "rdflib", "docopt"] +install_req = ["lxml", "pyyaml==3.13", "rdflib", "docopt"] if sys.version_info < (3, 4): install_req += ["enum34"]