Merge pull request #58 from pyexcel/v0.5.10

V0.5.10

CHANGELOG.rst

@@ -1,6 +1,18 @@
 Change log
 ================================================================================
 
+0.5.10 - 27.11.2018
+--------------------------------------------------------------------------------
+
+added
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. `#57 <https://github.com/pyexcel/pyexcel-io/issues/57>`_, long type will not
+   be written in ods. please use string type. And if the integer is equal or
+   greater than 10 to the power of 16, it will not be written either in ods. In
+   both situation, IntegerPrecisionLossError will be raised. And this version
+   enables pyexcel-ods and pyexcel-ods3 to do so.
+
 0.5.9.1 - 30.08.2018
 --------------------------------------------------------------------------------
 

changelog.yml

@@ -1,6 +1,12 @@
 name: pyexcel-io
 organisation: pyexcel
 releases:
+- changes:
+  - action: added
+    details:
+    - '`#57`, long type will not be written in ods. please use string type. And if the integer is equal or greater than 10 to the power of 16, it will not be written either in ods. In both situation, IntegerPrecisionLossError will be raised. And this version enables pyexcel-ods and pyexcel-ods3 to do so.'
+  date: 27.11.2018
+  version: 0.5.10
 - changes:
   - action: updated
     details:

docs/source/conf.py

@@ -29,9 +29,9 @@ copyright = u'2015-2018 Onni Software Ltd.'
 author = u'C.W.'
 
 # The short X.Y version
-version = u'0.5.9.1'
+version = u'0.5.10'
 # The full version, including alpha/beta/rc tags
-release = u'0.5.9.1'
+release = u'0.5.10'
 
 
 # -- General configuration ---------------------------------------------------
@@ -43,12 +43,7 @@ release = u'0.5.9.1'
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.doctest',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.viewcode',
-]
+extensions = [    'sphinx.ext.autodoc',    'sphinx.ext.doctest',    'sphinx.ext.intersphinx',    'sphinx.ext.viewcode',]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -75,7 +70,7 @@ language = 'en'
 exclude_patterns = []
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = None
 
 
 # -- Options for HTML output -------------------------------------------------

pyexcel-io.yml

@@ -1,9 +1,9 @@
 overrides: "pyexcel.yaml"
 name: "pyexcel-io"
 nick_name: io
-version: 0.5.9.1
-current_version: 0.5.9.1
-release: 0.5.9.1
+version: 0.5.10
+current_version: 0.5.10
+release: 0.5.10
 dependencies:
   - ordereddict;python_version<"2.7"
   - lml>=0.0.2

pyexcel_io/constants.py

@@ -59,3 +59,5 @@ SEPARATOR_FORMATTER = "---%s---" % DEFAULT_NAME + "%s"
 SEPARATOR_MATCHER = "---%s:(.*)---" % DEFAULT_NAME
 DEFAULT_CSV_STREAM_FILE_FORMATTER = "---%s:" % DEFAULT_NAME + "%s---%s"
 DEFAULT_CSV_NEWLINE = "\r\n"
+
+MAX_INTEGER = 999999999999999

pyexcel_io/exceptions.py

@@ -25,3 +25,32 @@ class UpgradePlugin(Exception):
     """raised when a known plugin is not compatible"""
 
     pass
+
+
+class IntegerAccuracyLossError(Exception):
+    """
+    When an interger is greater than 999999999999999, ods loses its accuracy.
+
+    from pyexcel import Sheet, get_sheet
+    s = Sheet()
+    s[0,0] = 999999999999999  # 15 '9's
+    print(s)
+    s.save_as('abc.ods')
+    b=get_sheet(file_name='abc.ods')
+    b[0,0] == s[0,0]
+
+    s = Sheet()
+    s[0,0] = 9999999999999999 # 16 '9's
+    print(s)
+    s.save_as('abc.ods')
+    b=get_sheet(file_name='abc.ods')
+    b[0,0] != s[0,0]
+    """
+    def __init__(self, message):
+        custom_message = (
+            message + '\n' +
+            "In order to keep its accuracy, please save as string. Then " +
+            "convert to int, long or float after the value will be read back"
+        )
+
+        super(IntegerAccuracyLossError, self).__init__(custom_message)

pyexcel_io/readers/csvr.py

@@ -84,7 +84,7 @@ class CSVMemoryMapIterator(compact.Iterator):
                 if bom_header == BOM_BIG_ENDIAN:
                     self.__endian = BIG_ENDIAN
         elif self.__endian == LITTLE_ENDIAN:
-            line = line[self.__zeros_left_in_2_row :]  # flake8: noqa
+            line = line[self.__zeros_left_in_2_row :]  # noqa: E203
         if self.__endian == LITTLE_ENDIAN:
             line = line.rstrip()
         line = line.decode(self.__encoding)

pyexcel_io/service.py

@@ -12,6 +12,8 @@ import math
 import datetime
 
 from pyexcel_io._compact import PY2
+from pyexcel_io import constants
+from pyexcel_io import exceptions
 
 
 def has_no_digits_in_float(value):
@@ -127,7 +129,7 @@ def time_value(value):
     """convert to time value accroding the specification"""
     import re
 
-    results = re.match("PT(\d+)H(\d+)M(\d+)S", value)
+    results = re.match(r"PT(\d+)H(\d+)M(\d+)S", value)
     if results and len(results.groups()) == 3:
         hour = int(results.group(1))
         minute = int(results.group(2))
@@ -177,7 +179,9 @@ ODS_WRITE_FORMAT_COVERSION = {
 }
 
 if PY2:
-    ODS_WRITE_FORMAT_COVERSION[unicode] = "string"
+    ODS_WRITE_FORMAT_COVERSION[unicode] = "string"  # noqa: F821
+    ODS_WRITE_FORMAT_COVERSION[long] = "throw_exception"  # noqa: F821
+
 
 VALUE_CONVERTERS = {
     "float": float_value,
@@ -189,6 +193,16 @@ VALUE_CONVERTERS = {
 }
 
 
+def throw_exception(value):
+    raise exceptions.IntegerAccuracyLossError("%s is too big" % value)
+
+
+def ods_float_value(value):
+    if value > constants.MAX_INTEGER:
+        raise exceptions.IntegerAccuracyLossError("%s is too big" % value)
+    return value
+
+
 def ods_date_value(value):
     return value.strftime("%Y-%m-%d")
 
@@ -219,6 +233,8 @@ ODS_VALUE_CONVERTERS = {
     "time": ods_time_value,
     "boolean": ods_bool_value,
     "timedelta": ods_timedelta_value,
+    "float": ods_float_value,
+    "throw_exception": throw_exception
 }
 
 

setup.py

@@ -5,13 +5,15 @@ import os
 import sys
 import codecs
 from shutil import rmtree
-from setuptools import setup, find_packages, Command
+
+from setuptools import Command, setup, find_packages
+
 PY2 = sys.version_info[0] == 2
 PY26 = PY2 and sys.version_info[1] < 7
 
 NAME = 'pyexcel-io'
 AUTHOR = 'C.W.'
-VERSION = '0.5.9.1'
+VERSION = '0.5.10'
 EMAIL = 'wangc_2011@hotmail.com'
 LICENSE = 'New BSD'
 DESCRIPTION = (
@@ -19,9 +21,10 @@ DESCRIPTION = (
     'format and to/from databases'
 )
 URL = 'https://github.com/pyexcel/pyexcel-io'
-DOWNLOAD_URL = '%s/archive/0.5.9.1.tar.gz' % URL
+DOWNLOAD_URL = '%s/archive/0.5.10.tar.gz' % URL
 FILES = ['README.rst', 'CHANGELOG.rst']
 KEYWORDS = [
+    'python',
     'API',
     'tsv',
     'tsvz',
@@ -29,7 +32,6 @@ KEYWORDS = [
     'csvz',
     'django',
     'sqlalchemy',
-    'python',
 ]
 
 CLASSIFIERS = [
@@ -63,8 +65,8 @@ EXTRAS_REQUIRE = {
 # You do not need to read beyond this line
 PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format(
     sys.executable)
-GS_COMMAND = ('gs pyexcel-io v0.5.9.1 ' +
-              "Find 0.5.9.1 in changelog for more details")
+GS_COMMAND = ('gs pyexcel-io v0.5.10 ' +
+              "Find 0.5.10 in changelog for more details")
 NO_GS_MESSAGE = ('Automatic github release is disabled. ' +
                  'Please install gease to enable it.')
 UPLOAD_FAILED_MSG = (

tests/test_service.py

@@ -2,6 +2,12 @@ from nose.tools import eq_, raises
 from pyexcel_io.service import date_value, time_value
 from pyexcel_io.service import detect_int_value
 from pyexcel_io.service import detect_float_value
+from pyexcel_io.service import ODS_WRITE_FORMAT_COVERSION
+from pyexcel_io.service import ods_float_value
+from pyexcel_io.service import throw_exception
+from pyexcel_io._compact import PY2
+from pyexcel_io.exceptions import IntegerAccuracyLossError
+from nose import SkipTest
 
 
 def test_date_util_parse():
@@ -89,3 +95,21 @@ def test_detect_float_value_on_custom_nan_text():
 def test_detect_float_value_on_custom_nan_text2():
     result = detect_float_value("nan", default_float_nan="nan")
     eq_(str(result), "nan")
+
+
+def test_ods_write_format_conversion():
+    if PY2:
+        expected = ODS_WRITE_FORMAT_COVERSION[long]  # noqa: F821
+        eq_('throw_exception', expected)
+    else:
+        raise SkipTest()
+
+
+@raises(IntegerAccuracyLossError)
+def test_big_int_value():
+    ods_float_value(1000000000000000)
+
+
+@raises(IntegerAccuracyLossError)
+def test_throw_exception():
+    throw_exception(1000000000000000)