From 4eade6920145b5953a42e4bbb14deda81dafa090 Mon Sep 17 00:00:00 2001 From: Eduard Braun Date: Tue, 8 Dec 2015 23:38:06 +0100 Subject: [PATCH] Open input file in binary mode an let XML parser deal with encoding. Fixes #26 --- scour/scour.py | 4 ++-- testscour.py | 12 +++++++++--- unittests/utf8.svg | 20 +++++++++++++++++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/scour/scour.py b/scour/scour.py index 335bd0e..a62ee58 100644 --- a/scour/scour.py +++ b/scour/scour.py @@ -3097,7 +3097,7 @@ def scourString(in_string, options=None): # input is a filename # returns the minidom doc representation of the SVG def scourXmlFile(filename, options=None): - with open(filename) as f: + with open(filename, "rb") as f: in_string = f.read() out_string = scourString(in_string, options) return xml.dom.minidom.parseString(out_string.encode('utf-8')) @@ -3235,7 +3235,7 @@ def parse_args(args=None, ignore_additional_args=False): _options_parser.error("Input filename is the same as output filename") if options.infilename: - infile = maybe_gziped_file(options.infilename) + infile = maybe_gziped_file(options.infilename, "rb") # GZ: could catch a raised IOError here and report else: # GZ: could sniff for gzip compression here diff --git a/testscour.py b/testscour.py index 7b29bdf..8912288 100755 --- a/testscour.py +++ b/testscour.py @@ -604,12 +604,18 @@ class ChangeQuadToShorthandInPath(unittest.TestCase): self.assertEqual(path.getAttribute('d'), 'm10 100q50-50 100 0t100 0', 'Did not change quadratic curves into shorthand curve segments in path') -class HandleNonAsciiUtf8(unittest.TestCase): +class HandleUTF8(unittest.TestCase): def runTest(self): doc = scour.scourXmlFile('unittests/utf8.svg') + text = u'Hello in many languages:\nar: أهلا\nbn: হ্যালো\nel: Χαίρετε\nen: Hello\nhi: नमस्ते\niw: שלום\nja: こんにちは\nkm: ជំរាបសួរ\nml: ഹലോ\nru: Здравствуйте\nur: ہیلو\nzh: 您好' desc = six.text_type(doc.getElementsByTagNameNS(SVGNS, 'desc')[0].firstChild.wholeText).strip() - self.assertEqual( desc, u'ú', - 'Did not handle non-ASCII characters' ) + self.assertEqual( desc, text, 'Did not handle international UTF8 characters' ) + desc = six.text_type(doc.getElementsByTagNameNS(SVGNS, 'desc')[1].firstChild.wholeText).strip() + self.assertEqual( desc, u'“”‘’–—…‐‒°©®™•½¼¾⅓⅔†‡µ¢£€«»♠♣♥♦¿�', 'Did not handle common UTF8 characters' ) + desc = six.text_type(doc.getElementsByTagNameNS(SVGNS, 'desc')[2].firstChild.wholeText).strip() + self.assertEqual( desc, u':-×÷±∞π∅≤≥≠≈∧∨∩∪∈∀∃∄∑∏←↑→↓↔↕↖↗↘↙↺↻⇒⇔', 'Did not handle mathematical UTF8 characters' ) + desc = six.text_type(doc.getElementsByTagNameNS(SVGNS, 'desc')[3].firstChild.wholeText).strip() + self.assertEqual( desc, u'⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁽⁾ⁿⁱ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎', 'Did not handle superscript/subscript UTF8 characters' ) class HandleSciNoInPathData(unittest.TestCase): def runTest(self): diff --git a/unittests/utf8.svg b/unittests/utf8.svg index 6c77d7a..dd63f12 100644 --- a/unittests/utf8.svg +++ b/unittests/utf8.svg @@ -1,5 +1,19 @@ - - ú + + Hello in many languages: +ar: أهلا +bn: হ্যালো +el: Χαίρετε +en: Hello +hi: नमस्ते +iw: שלום +ja: こんにちは +km: ជំរាបសួរ +ml: ഹലോ +ru: Здравствуйте +ur: ہیلو +zh: 您好 + “”‘’–—…‐‒°©®™•½¼¾⅓⅔†‡µ¢£€«»♠♣♥♦¿� + :-×÷±∞π∅≤≥≠≈∧∨∩∪∈∀∃∄∑∏←↑→↓↔↕↖↗↘↙↺↻⇒⇔ + ⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁽⁾ⁿⁱ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎