From 5a7c35518b4e1df2a3f94e33fded95e6a5a72c0c Mon Sep 17 00:00:00 2001 From: Kunal Khandelwal Date: Mon, 25 Jun 2018 17:12:59 +0530 Subject: [PATCH 1/2] added unicode support for TSV for pytjhon 2 --- import_export/formats/base_formats.py | 6 ++++++ tests/core/exports/books-unicode.tsv | 2 ++ tests/core/tests/test_base_formats.py | 17 +++++++++++++++++ tests/database.db | Bin 70656 -> 70656 bytes 4 files changed, 25 insertions(+) create mode 100644 tests/core/exports/books-unicode.tsv diff --git a/import_export/formats/base_formats.py b/import_export/formats/base_formats.py index fb71b11..7933760 100644 --- a/import_export/formats/base_formats.py +++ b/import_export/formats/base_formats.py @@ -164,6 +164,12 @@ class TSV(TextFormat): TABLIB_MODULE = 'tablib.formats._tsv' CONTENT_TYPE = 'text/tab-separated-values' + def create_dataset(self, in_stream, **kwargs): + if sys.version_info[0] < 3: + # python 2.7 csv does not do unicode + return super(TSV, self).create_dataset(in_stream.encode('utf-8'), **kwargs) + return super(TSV, self).create_dataset(in_stream, **kwargs) + class ODS(TextFormat): TABLIB_MODULE = 'tablib.formats._ods' diff --git a/tests/core/exports/books-unicode.tsv b/tests/core/exports/books-unicode.tsv new file mode 100644 index 0000000..e586724 --- /dev/null +++ b/tests/core/exports/books-unicode.tsv @@ -0,0 +1,2 @@ +id name author_email +1 Some bookš test@example.com diff --git a/tests/core/tests/test_base_formats.py b/tests/core/tests/test_base_formats.py index 8245436..003b633 100644 --- a/tests/core/tests/test_base_formats.py +++ b/tests/core/tests/test_base_formats.py @@ -74,3 +74,20 @@ class CSVTest(TestCase): with open(filename, self.format.get_read_mode()) as in_stream: data = force_text(in_stream.read()) base_formats.CSV().create_dataset(data) + + +class TSVTest(TestCase): + + def setUp(self): + self.format = base_formats.CSV() + + def test_import_unicode(self): + # importing tsv UnicodeEncodeError + filename = os.path.join( + os.path.dirname(__file__), + os.path.pardir, + 'exports', + 'books-unicode.tsv') + with open(filename, self.format.get_read_mode()) as in_stream: + data = force_text(in_stream.read()) + base_formats.CSV().create_dataset(data) diff --git a/tests/database.db b/tests/database.db index 29e644c20c374a9fb4ca410b262fe2fce129a851..73a74853370b43cf79b4dd79b64257a137d7fec2 100644 GIT binary patch delta 435 zcmZoz!P2mTWr8%L&qNt#Rv!l4k2^M|EI7@<#Mr{XbcnHKvm!?YV@#7Z6RU3vt7D3B zN@8+uhKZqxWkqgwrJ-4lxlvVBT6Srsd5WP?UZ%ghuZeF}RCZK|TV_yd%RF0p!SC(&;Yq?)|RK{ev^Kz!1<&ojOMp5perhZwzrhY)S zZ&tdYUwEWpRIXPp(1>!Nc$II6Q>JgOr?FpFx~Z>=@8rVs74jCk2Ijg3mI?-DR>me) zhK71Z<`(7_hE3LtU>8q*a9LsVt8-F}98ApH8JO=dZ{I8^vzl2|h1r*pkZ+hK-?^*- z)OVPH`4RJBpuX+Q@^Z|Mj7A2AIBaFw{OI~r^+;ng0|NtSL5WcD8E9lJf9@v1xNrc6ydRCpvwjU0>S_S!m}X=xB>`*I{^hvgatv9 z;L{+p>dqPg1_1-y00Z2!5ggJ3librN1_1-&00ZK)5ggb9v*Op4Cs{HyFfcGAOm%N> zM@&*NRcvu)Bsn&7LOFCxZFo3AX<9EbYj;|DI8aeCct|U7bwxNsOgVCNOhR)xWobf5 jJu)ycHZ3qTEig79Ff%$dGdeOeE;%_gIWso1(El(Ln^!ur From 2e9195f953a35d4bf5a46d2cbbff8de597aef125 Mon Sep 17 00:00:00 2001 From: Kunal Khandelwal Date: Mon, 25 Jun 2018 18:03:18 +0530 Subject: [PATCH 2/2] added test for tsv import --- tests/core/exports/books-mac.tsv | 2 ++ tests/core/tests/test_base_formats.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 tests/core/exports/books-mac.tsv diff --git a/tests/core/exports/books-mac.tsv b/tests/core/exports/books-mac.tsv new file mode 100644 index 0000000..1cc5de4 --- /dev/null +++ b/tests/core/exports/books-mac.tsv @@ -0,0 +1,2 @@ +id name author_email +1 Some book test@example.com diff --git a/tests/core/tests/test_base_formats.py b/tests/core/tests/test_base_formats.py index 003b633..3a59377 100644 --- a/tests/core/tests/test_base_formats.py +++ b/tests/core/tests/test_base_formats.py @@ -79,7 +79,18 @@ class CSVTest(TestCase): class TSVTest(TestCase): def setUp(self): - self.format = base_formats.CSV() + self.format = base_formats.TSV() + + def test_import_mac(self): + filename = os.path.join( + os.path.dirname(__file__), + os.path.pardir, + 'exports', + 'books-mac.tsv') + with open(filename, self.format.get_read_mode()) as in_stream: + actual = in_stream.read() + expected = 'id\tname\tauthor_email\n1\tSome book\ttest@example.com\n' + self.assertEqual(actual, expected) def test_import_unicode(self): # importing tsv UnicodeEncodeError @@ -90,4 +101,4 @@ class TSVTest(TestCase): 'books-unicode.tsv') with open(filename, self.format.get_read_mode()) as in_stream: data = force_text(in_stream.read()) - base_formats.CSV().create_dataset(data) + base_formats.TSV().create_dataset(data)