Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Fix VersionConverter encoding issue #296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
achilleas-k merged 7 commits into from
Jul 25, 2018
Merged

Fix VersionConverter encoding issue #296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3a21f0f
[tools/versionConv] Fix encoding issue
mpsonntag Jul 24, 2018
eed09df
[tools/xmlparser] Fix from_csv single str val read
mpsonntag Jul 25, 2018
9fa62d8
[tools/xmlparser] Strip ws and quotes on to_csv
mpsonntag Jul 25, 2018
4e134aa
[tools/versionConv] Save multivals w/o whitespace
mpsonntag Jul 25, 2018
40bc156
[test/versionConv] Fix changed multival save
mpsonntag Jul 25, 2018
35161bc
[travis] Unsupport py3.5 on macOS
mpsonntag Jul 25, 2018
46a4fbb
[test/versionConv] Add strng value tests
mpsonntag Jul 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,6 @@ matrix:
- os: linux
python: "3.6"

- os: osx
language: generic
env:
- OSXENV=3.5.0
- os: osx
language: generic
env:
Expand Down
5 changes: 4 additions & 1 deletion odml/tools/version_converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ def _parse_xml(self):
if elem in doc:
doc = doc.replace(elem, val)

# Make sure encoding is present for the xml parser
doc = doc.encode('utf-8')

# Make pretty print available by resetting format
parser = ET.XMLParser(remove_blank_text=True)
tree = ET.ElementTree(ET.fromstring(doc, parser))
Expand Down Expand Up @@ -300,7 +303,7 @@ def _handle_properties(self, root):

if value.text:
if main_val.text:
main_val.text += ", " + value.text.strip()
main_val.text += "," + value.text.strip()
multiple_values = True
else:
main_val.text = value.text.strip()
Expand Down
16 changes: 13 additions & 3 deletions odml/tools/xmlparser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,15 @@


def to_csv(val):
unicode_values = list(map(unicode, val))
# Make sure all individual values do not contain
# leading or trailing whitespaces.
unicode_values = list(map(unicode.strip, map(unicode, val)))
stream = StringIO()
writer = csv.writer(stream, dialect="excel")
writer.writerow(unicode_values)
csv_string = stream.getvalue().strip()
# Strip any csv.writer added carriage return line feeds
# and double quotes before saving.
csv_string = stream.getvalue().strip().strip('"')
if len(unicode_values) > 1:
csv_string = "[" + csv_string + "]"
return csv_string
Expand All @@ -40,8 +44,14 @@ def to_csv(val):
def from_csv(value_string):
if not value_string:
return []
if value_string[0] == "[":
if value_string[0] == "[" and value_string[-1] == "]":
value_string = value_string[1:-1]
else:
# This is a single string entry, any comma contained
# is part of the value and must not be used to
# split up the string.
return [value_string]

Copy link
Member

@jgrewe jgrewe Jul 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

Copy link
Contributor Author

@mpsonntag mpsonntag Jul 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaSprenger caught that one, I'm just taking it one step further. ;)

if not value_string:
return []
stream = StringIO(value_string)
Expand Down
68 changes: 64 additions & 4 deletions test/test_version_converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ def test_convert_odml_file(self):
self.assertEqual(val_elems[0].find("unit"), None)
self.assertEqual(val_elems[0].find("type"), None)
self.assertEqual(val_elems[0].find("uncertainty"), None)
self.assertEqual(val_elems[0].text, "[0, 45]")
self.assertEqual(val_elems[0].text, "[0,45]")
self.assertEqual(prop.find("unit").text, "deg")
self.assertEqual(len(prop.findall("unit")), 1)
self.assertEqual(prop.find("type").text, "int")
Expand Down Expand Up @@ -481,6 +481,37 @@ def test_convert_odml_file_value(self):
</value>
</property>

<property>
<value>Single, string, value, with, many, commata.<type>string</type></value>
<name>testSingleString</name>
</property>

<property>
<value>A<type>string</type></value>
<value>B<type>string</type></value>
<value>C<type>string</type></value>
<name>testStringList</name>
</property>

<property>
<value> Single string value with wrapping whitespace <type>string</type></value>
<name>testStringWhiteSpace</name>
</property>

<property>
<value> Multiple Strings <type>string</type></value>
<value> with wrapping <type>string</type></value>
<value> Whitespace <type>string</type></value>
<name>testStringListWhiteSpace</name>
</property>

<property>
<value> 1 <type>int</type></value>
<value> 2 <type>int</type></value>
<value> 3 <type>int</type></value>
<name>testIntListWhiteSpace</name>
</property>

</section>
</odML>
"""
Expand All @@ -490,7 +521,7 @@ def test_convert_odml_file_value(self):
conv_doc = vc._convert(vc._parse_xml())
root = conv_doc.getroot()
sec = root.find("section")
self.assertEqual(len(sec), 9)
self.assertEqual(len(sec), 14)

# Test single value export
prop = sec.findall("property")[0]
Expand All @@ -500,7 +531,7 @@ def test_convert_odml_file_value(self):
# Test multiple value export
prop = sec.findall("property")[1]
self.assertEqual(len(prop), 2)
self.assertEqual(prop.find("value").text, "[1, 2, 3]")
self.assertEqual(prop.find("value").text, "[1,2,3]")

# Test empty value export
prop = sec.findall("property")[2]
Expand All @@ -521,7 +552,7 @@ def test_convert_odml_file_value(self):
# Test valid multiple Value tag export
prop = sec.findall("property")[4]
self.assertEqual(len(prop), 7)
self.assertEqual(prop.find("value").text, "[0.1, 0.2, 3]")
self.assertEqual(prop.find("value").text, "[0.1,0.2,3]")
self.assertEqual(prop.find("type").text, "float")
self.assertEqual(prop.find("uncertainty").text, "0.05")
self.assertEqual(prop.find("unit").text, "mV")
Expand All @@ -541,6 +572,35 @@ def test_convert_odml_file_value(self):
self.assertEqual(prop.find("name").text, "Unsupported binary value dtype replace")
self.assertEqual(prop.find("type").text, "text")

# Test single string value with commata
prop = sec.findall("property")[8]
self.assertEqual(prop.find("name").text, "testSingleString")
self.assertEqual(prop.find("value").text,
"Single, string, value, with, many, commata.")

# Test string list import
prop = sec.findall("property")[9]
self.assertEqual(prop.find("name").text, "testStringList")
self.assertEqual(prop.find("value").text, "[A,B,C]")

# Test single string values wrapping whitespace removal
prop = sec.findall("property")[10]
self.assertEqual(prop.find("name").text, "testStringWhiteSpace")
self.assertEqual(prop.find("value").text,
"Single string value with wrapping whitespace")

# Test multiple string values with wrapping whitespace removal
prop = sec.findall("property")[11]
self.assertEqual(prop.find("name").text, "testStringListWhiteSpace")
self.assertEqual(prop.find("value").text,
"[Multiple Strings,with wrapping,Whitespace]")

# Test multiple int values with wrapping whitespaces
prop = sec.findall("property")[12]
self.assertEqual(prop.find("name").text, "testIntListWhiteSpace")
self.assertEqual(prop.find("type").text, "int")
self.assertEqual(prop.find("value").text, "[1,2,3]")

def test_parse_dict_document(self):
# Test appending tags; not appending empty sections
doc_dict = {'Document': {'author': 'HPL', 'sections': []}}
Expand Down