diff --git a/.moban.d/README.rst b/.moban.d/README.rst index d17295c..83502eb 100644 --- a/.moban.d/README.rst +++ b/.moban.d/README.rst @@ -1,5 +1,8 @@ {%extends 'README.rst.jj2' %} +{%block documentation_link%} +{%endblock%} + {%block description%} **pyexcel-ods** is a tiny wrapper library to read, manipulate and write data in ods format using python 2.6 and python 2.7. You are likely to use it with diff --git a/.moban.d/tests/requirements.txt b/.moban.d/tests/requirements.txt index 3f0c5aa..962ea54 100644 --- a/.moban.d/tests/requirements.txt +++ b/.moban.d/tests/requirements.txt @@ -1,5 +1,6 @@ {% extends 'tests/requirements.txt.jj2' %} {%block extras %} +psutil pyexcel pyexcel-xls {%endblock%} diff --git a/.travis.yml b/.travis.yml index 23877d2..3f300e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,13 @@ python: - 2.7 - 2.6 before_install: + - cd $HOME + - "if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then deactivate && wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-5.7.1-linux_x86_64-portable.tar.bz2 -O - | tar -jxf - && echo 'Setting up aliases...' && ln -s pypy-5.7.1-linux_x86_64-portable pypy2-latest && export PATH=$HOME/pypy2-latest/bin/:$PATH && virtualenv --no-site-packages --python ~/pypy2-latest/bin/pypy pypy2-env && echo 'Creating custom env...' && source pypy2-env/bin/activate && python -V; fi" + - cd - - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then mv min_requirements.txt requirements.txt ; fi - - pip install --upgrade setuptools "pip==7.1" - test ! -f rnd_requirements.txt || pip install --no-deps -r rnd_requirements.txt - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; - pip install -r tests/requirements.txt diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 85cf8eb..8ee7eb0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,7 +24,7 @@ Updated 0.3.1 - 02.02.2017 -------------------------------------------------------------------------------- -Updated +Added ******************************************************************************** #. Recognize currency type diff --git a/README.rst b/README.rst index 2e9179a..16ef068 100644 --- a/README.rst +++ b/README.rst @@ -11,6 +11,7 @@ pyexcel-ods - Let you focus on data, instead of ods format .. image:: https://codecov.io/github/pyexcel/pyexcel-ods/coverage.png :target: https://codecov.io/github/pyexcel/pyexcel-ods + **pyexcel-ods** is a tiny wrapper library to read, manipulate and write data in ods format using python 2.6 and python 2.7. You are likely to use it with `pyexcel `_. @@ -41,6 +42,21 @@ or clone it and install it: $ cd pyexcel-ods $ python setup.py install +Support the project +================================================================================ + +If your company has embedded pyexcel and its components into a revenue generating +product, please `support me on patreon `_ to +maintain the project and develop it further. + +If you are an individual, you are welcome to support me too on patreon and for however long +you feel like to. As a patreon, you will receive +`early access to pyexcel related contents `_. + +With your financial support, I will be able to invest +a little bit more time in coding, documentation and writing interesting posts. + + Usage ================================================================================ @@ -345,10 +361,3 @@ ODSReader is originally written by `Marco Conti >> import os >>> os.unlink("your_file.ods") >>> os.unlink("another_file.ods") - -Support the project -================================================================================ - -If your company has embedded pyexcel and its components into a revenue generating -product, please `support me on patreon `_ to -maintain the project and develop it further. diff --git a/docs/source/conf.py b/docs/source/conf.py index 6c5de13..01ee293 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,7 +11,7 @@ extensions = [ ] intersphinx_mapping = { - 'pyexcel': ('http://pyexcel.readthedocs.org/en/latest/', None) + 'pyexcel': ('http://pyexcel.readthedocs.org/en/latest/', None), } spelling_word_list_filename = 'spelling_wordlist.txt' templates_path = ['_templates'] @@ -20,8 +20,8 @@ master_doc = 'index' project = u'pyexcel-ods' copyright = u'2015-2017 Onni Software Ltd.' -version = '0.3.3' -release = '0.3.3' +version = '0.4.0' +release = '0.4.0' exclude_patterns = [] pygments_style = 'sphinx' html_theme = 'default' diff --git a/pyexcel_ods.yaml b/pyexcel_ods.yaml index c6123b8..b780f92 100644 --- a/pyexcel_ods.yaml +++ b/pyexcel_ods.yaml @@ -1,10 +1,10 @@ overrides: "pyexcel.yaml" name: "pyexcel-ods" nick_name: ods -version: 0.3.3 -release: 0.3.3 +version: 0.4.0 +release: 0.4.0 file_type: ods dependencies: - - pyexcel-io>=0.3.0 + - pyexcel-io>=0.4.0 - odfpy>=1.3.3 description: A wrapper library to read, manipulate and write data in ods format diff --git a/pyexcel_ods/__init__.py b/pyexcel_ods/__init__.py index 3d0a58f..70c50de 100644 --- a/pyexcel_ods/__init__.py +++ b/pyexcel_ods/__init__.py @@ -8,17 +8,20 @@ # flake8: noqa # this line has to be place above all else # because of dynamic import -__FILE_TYPE__ = 'ods' -__META__ = { - 'submodule': __FILE_TYPE__, - 'file_types': [__FILE_TYPE__], - 'stream_type': 'binary' -} -__pyexcel_io_plugins__ = [__META__] - - +from pyexcel_io.plugins import IOPluginInfoChain from pyexcel_io.io import get_data as read_data, isstream, store_data as write_data +__FILE_TYPE__ = 'ods' +IOPluginInfoChain(__name__).add_a_reader( + relative_plugin_class_path='odsr.ODSBook', + file_types=[__FILE_TYPE__], + stream_type='binary' +).add_a_writer( + relative_plugin_class_path='odsw.ODSWriter', + file_types=[__FILE_TYPE__], + stream_type='binary' +) + def save_data(afile, data, file_type=None, **keywords): """standalone module function for writing module supported file type""" diff --git a/pyexcel_ods/ods.py b/pyexcel_ods/odsr.py similarity index 67% rename from pyexcel_ods/ods.py rename to pyexcel_ods/odsr.py index 10f3525..b37d109 100644 --- a/pyexcel_ods/ods.py +++ b/pyexcel_ods/odsr.py @@ -1,3 +1,12 @@ +""" + pyexcel_ods.odsr + ~~~~~~~~~~~~~~~~~~~~~ + + ods reader + + :copyright: (c) 2014-2017 by Onni Software Ltd. + :license: New BSD License, see LICENSE for more details +""" # Copyright 2011 Marco Conti # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,27 +22,19 @@ # limitations under the License. # Thanks to grt for the fixes -import sys import math from odf.table import TableRow, TableCell, Table from odf.text import P from odf.namespaces import OFFICENS -from odf.opendocument import OpenDocumentSpreadsheet, load +from odf.opendocument import load -from pyexcel_io.book import BookReader, BookWriter -from pyexcel_io.sheet import SheetReader, SheetWriter +from pyexcel_io.book import BookReader +from pyexcel_io.sheet import SheetReader +from pyexcel_io._compact import OrderedDict, PY2 import pyexcel_ods.converter as converter -PY2 = sys.version_info[0] == 2 - -PY27_BELOW = PY2 and sys.version_info[1] < 7 -if PY27_BELOW: - from ordereddict import OrderedDict -else: - from collections import OrderedDict - class ODSSheet(SheetReader): """native ods sheet""" @@ -115,7 +116,6 @@ class ODSSheet(SheetReader): class ODSBook(BookReader): """read ods book""" - def open(self, file_name, **keywords): """open ods file""" BookReader.open(self, file_name, **keywords) @@ -160,6 +160,9 @@ class ODSBook(BookReader): sheet = ODSSheet(native_sheet, **self._keywords) return {sheet.name: sheet.to_array()} + def close(self): + self._native_book = None + def _load_from_memory(self): self._native_book = load(self._file_stream) @@ -167,92 +170,6 @@ class ODSBook(BookReader): self._native_book = load(self._file_name) -class ODSSheetWriter(SheetWriter): - """ - ODS sheet writer - """ - def set_sheet_name(self, name): - """initialize the native table""" - self._native_sheet = Table(name=name) - - def set_size(self, size): - """not used in this class but used in ods3""" - pass - - def write_cell(self, row, cell): - """write a native cell""" - cell_to_be_written = TableCell() - cell_type = type(cell) - cell_odf_type = converter.ODS_WRITE_FORMAT_COVERSION.get( - cell_type, "string") - cell_to_be_written.setAttrNS(OFFICENS, "value-type", cell_odf_type) - cell_odf_value_token = converter.VALUE_TOKEN.get( - cell_odf_type, "value") - converter_func = converter.ODS_VALUE_CONVERTERS.get( - cell_odf_type, None) - if converter_func: - cell = converter_func(cell) - if cell_odf_type != 'string': - cell_to_be_written.setAttrNS(OFFICENS, cell_odf_value_token, cell) - cell_to_be_written.addElement(P(text=cell)) - else: - lines = cell.split('\n') - for line in lines: - cell_to_be_written.addElement(P(text=line)) - row.addElement(cell_to_be_written) - - def write_row(self, array): - """ - write a row into the file - """ - row = TableRow() - self._native_sheet.addElement(row) - for cell in array: - self.write_cell(row, cell) - - def close(self): - """ - This call writes file - - """ - self._native_book.spreadsheet.addElement(self._native_sheet) - - -class ODSWriter(BookWriter): - """ - open document spreadsheet writer - - """ - def __init__(self): - BookWriter.__init__(self) - self._native_book = OpenDocumentSpreadsheet() - - def create_sheet(self, name): - """ - write a row into the file - """ - return ODSSheetWriter(self._native_book, None, name) - - def close(self): - """ - This call writes file - - """ - self._native_book.write(self._file_alike_object) - - def is_integer_ok_for_xl_float(value): """check if a float had zero value in digits""" return value == math.floor(value) - - -_ods_registry = { - "file_type": "ods", - "reader": ODSBook, - "writer": ODSWriter, - "stream_type": "binary", - "mime_type": "application/vnd.oasis.opendocument.spreadsheet", - "library": "pyexcel-ods" -} - -exports = (_ods_registry,) diff --git a/pyexcel_ods/odsw.py b/pyexcel_ods/odsw.py new file mode 100644 index 0000000..87856c5 --- /dev/null +++ b/pyexcel_ods/odsw.py @@ -0,0 +1,99 @@ +""" + pyexcel_ods.odsw + ~~~~~~~~~~~~~~~~~~~~~ + + ods writer + + :copyright: (c) 2014-2017 by Onni Software Ltd. + :license: New BSD License, see LICENSE for more details +""" +import sys + +from odf.table import TableRow, TableCell, Table +from odf.text import P +from odf.namespaces import OFFICENS +from odf.opendocument import OpenDocumentSpreadsheet + +from pyexcel_io.book import BookWriter +from pyexcel_io.sheet import SheetWriter + +import pyexcel_ods.converter as converter + +PY2 = sys.version_info[0] == 2 + +PY27_BELOW = PY2 and sys.version_info[1] < 7 + + +class ODSSheetWriter(SheetWriter): + """ + ODS sheet writer + """ + def set_sheet_name(self, name): + """initialize the native table""" + self._native_sheet = Table(name=name) + + def set_size(self, size): + """not used in this class but used in ods3""" + pass + + def write_cell(self, row, cell): + """write a native cell""" + cell_to_be_written = TableCell() + cell_type = type(cell) + cell_odf_type = converter.ODS_WRITE_FORMAT_COVERSION.get( + cell_type, "string") + cell_to_be_written.setAttrNS(OFFICENS, "value-type", cell_odf_type) + cell_odf_value_token = converter.VALUE_TOKEN.get( + cell_odf_type, "value") + converter_func = converter.ODS_VALUE_CONVERTERS.get( + cell_odf_type, None) + if converter_func: + cell = converter_func(cell) + if cell_odf_type != 'string': + cell_to_be_written.setAttrNS(OFFICENS, cell_odf_value_token, cell) + cell_to_be_written.addElement(P(text=cell)) + else: + lines = cell.split('\n') + for line in lines: + cell_to_be_written.addElement(P(text=line)) + row.addElement(cell_to_be_written) + + def write_row(self, array): + """ + write a row into the file + """ + row = TableRow() + self._native_sheet.addElement(row) + for cell in array: + self.write_cell(row, cell) + + def close(self): + """ + This call writes file + + """ + self._native_book.spreadsheet.addElement(self._native_sheet) + + +class ODSWriter(BookWriter): + """ + open document spreadsheet writer + + """ + def __init__(self): + BookWriter.__init__(self) + self._native_book = OpenDocumentSpreadsheet() + + def create_sheet(self, name): + """ + write a row into the file + """ + return ODSSheetWriter(self._native_book, None, name) + + def close(self): + """ + This call writes file + + """ + self._native_book.write(self._file_alike_object) + self._native_book = None diff --git a/requirements.txt b/requirements.txt index 606b659..bbd7824 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -pyexcel-io>=0.3.0 +pyexcel-io>=0.4.0 odfpy>=1.3.3 diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 30a50e9..0e70294 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1 +1,4 @@ +https://github.com/chfw/lml/archive/master.zip https://github.com/pyexcel/pyexcel-io/archive/master.zip +https://github.com/pyexcel/pyexcel/archive/master.zip +https://github.com/pyexcel/pyexcel-xls/archive/master.zip diff --git a/setup.py b/setup.py index 2c36ee1..78ac3fa 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ except ImportError: NAME = 'pyexcel-ods' AUTHOR = 'C.W.' -VERSION = '0.3.3' +VERSION = '0.4.0' EMAIL = 'wangc_2011 (at) hotmail.com' LICENSE = 'New BSD' DESCRIPTION = ( @@ -36,7 +36,7 @@ CLASSIFIERS = [ ] INSTALL_REQUIRES = [ - 'pyexcel-io>=0.3.0', + 'pyexcel-io>=0.4.0', 'odfpy>=1.3.3', ] diff --git a/test.bat b/test.bat index 86e1379..7e4a6fa 100644 --- a/test.bat +++ b/test.bat @@ -1,4 +1,2 @@ - - pip freeze -nosetests --with-cov --cover-package pyexcel_ods --cover-package tests --with-doctest --doctest-extension=.rst tests README.rst docs/source pyexcel_ods && flake8 . --exclude=.moban.d --builtins=unicode,xrange,long +nosetests --with-cov --cover-package pyexcel_ods --cover-package tests --with-doctest --doctest-extension=.rst README.rst tests docs/source pyexcel_ods && flake8 . --exclude=.moban.d --builtins=unicode,xrange,long diff --git a/test.sh b/test.sh index 86e1379..7e4a6fa 100644 --- a/test.sh +++ b/test.sh @@ -1,4 +1,2 @@ - - pip freeze -nosetests --with-cov --cover-package pyexcel_ods --cover-package tests --with-doctest --doctest-extension=.rst tests README.rst docs/source pyexcel_ods && flake8 . --exclude=.moban.d --builtins=unicode,xrange,long +nosetests --with-cov --cover-package pyexcel_ods --cover-package tests --with-doctest --doctest-extension=.rst README.rst tests docs/source pyexcel_ods && flake8 . --exclude=.moban.d --builtins=unicode,xrange,long diff --git a/tests/base.py b/tests/base.py index f92845b..e469574 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,7 +1,7 @@ -import os +import os # noqa import pyexcel -import datetime -from nose.tools import raises, eq_ +import datetime # noqa +from nose.tools import raises, eq_ # noqa def create_sample_file1(file): diff --git a/tests/requirements.txt b/tests/requirements.txt index f13a9b5..4b23de1 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,8 @@ nose +mock;python_version<"3" codecov coverage flake8 +psutil pyexcel pyexcel-xls diff --git a/tests/test_bug_fixes.py b/tests/test_bug_fixes.py index 8bb936d..58443b9 100644 --- a/tests/test_bug_fixes.py +++ b/tests/test_bug_fixes.py @@ -1,12 +1,14 @@ #!/usr/bin/python # -*- encoding: utf-8 -*- import os +import psutil +import pyexcel as pe from pyexcel_ods import get_data, save_data from nose.tools import raises, eq_ def test_bug_fix_for_issue_1(): - data = get_data(os.path.join("tests", "fixtures", "repeated.ods")) + data = get_data(get_fixtures("repeated.ods")) eq_(data["Sheet1"], [['repeated', 'repeated', 'repeated', 'repeated']]) @@ -81,20 +83,53 @@ def test_issue_13(): def test_issue_14(): # pyexcel issue 61 test_file = "issue_61.ods" - data = get_data(os.path.join("tests", "fixtures", test_file), + data = get_data(get_fixtures(test_file), skip_empty_rows=True) eq_(data['S-LMC'], [[u'aaa'], [0]]) def test_issue_6(): test_file = "12_day_as_time.ods" - data = get_data(os.path.join("tests", "fixtures", test_file), + data = get_data(get_fixtures(test_file), skip_empty_rows=True) eq_(data['Sheet1'][0][0].days, 12) def test_issue_19(): test_file = "pyexcel_81_ods_19.ods" - data = get_data(os.path.join("tests", "fixtures", test_file), + data = get_data(get_fixtures(test_file), skip_empty_rows=True) eq_(data['product.template'][1][1], 'PRODUCT NAME PMP') + + +def test_issue_83_ods_file_handle(): + # this proves that odfpy + # does not leave a file handle open at all + proc = psutil.Process() + test_file = get_fixtures("issue_61.ods") + open_files_l1 = proc.open_files() + + # start with a csv file + data = pe.iget_array(file_name=test_file, library='pyexcel-ods') + open_files_l2 = proc.open_files() + delta = len(open_files_l2) - len(open_files_l1) + # cannot catch open file handle + assert delta == 0 + + # now the file handle get opened when we run through + # the generator + list(data) + open_files_l3 = proc.open_files() + delta = len(open_files_l3) - len(open_files_l1) + # cannot catch open file handle + assert delta == 0 + + # free the fish + pe.free_resources() + open_files_l4 = proc.open_files() + # this confirms that no more open file handle + eq_(open_files_l1, open_files_l4) + + +def get_fixtures(filename): + return os.path.join("tests", "fixtures", filename) diff --git a/tests/test_ods_reader.py b/tests/test_ods_reader.py index 551bda3..cf7c1f3 100644 --- a/tests/test_ods_reader.py +++ b/tests/test_ods_reader.py @@ -1,11 +1,12 @@ import os -from pyexcel_ods import ods +from pyexcel_ods.odsr import ODSBook +from pyexcel_ods.odsw import ODSWriter from base import ODSCellTypes class TestODSReader(ODSCellTypes): def setUp(self): - r = ods.ODSBook() + r = ODSBook() r.open(os.path.join("tests", "fixtures", "ods_formats.ods")) @@ -17,17 +18,17 @@ class TestODSReader(ODSCellTypes): class TestODSWriter(ODSCellTypes): def setUp(self): - r = ods.ODSBook() + r = ODSBook() r.open(os.path.join("tests", "fixtures", "ods_formats.ods")) self.data1 = r.read_all() self.testfile = "odswriter.ods" - w = ods.ODSWriter() + w = ODSWriter() w.open(self.testfile) w.write(self.data1) w.close() - r2 = ods.ODSBook() + r2 = ODSBook() r2.open(self.testfile) self.data = r2.read_all() for key in self.data.keys(): diff --git a/tests/test_writer.py b/tests/test_writer.py index 4eedec4..5dec2db 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -1,5 +1,6 @@ import os -from pyexcel_ods.ods import ODSWriter as Writer, ODSBook as Reader +from pyexcel_ods.odsw import ODSWriter as Writer +from pyexcel_ods.odsr import ODSBook as Reader from base import PyexcelWriterBase, PyexcelHatWriterBase @@ -18,10 +19,10 @@ class TestNativeODSWriter: reader = Reader() reader.open(self.testfile) content = reader.read_all() - reader.close() for key in content.keys(): content[key] = list(content[key]) assert content == self.content + reader.close() def tearDown(self): if os.path.exists(self.testfile):