From 41ef6d1310bb1328d9e310bb45785adf07313ef7 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Nov 2018 20:56:34 +0000 Subject: [PATCH 001/143] :hammer: code refactoring using isort and black --- .moban.d/docs/source/conf.py | 1 - .moban.d/setup.py | 1 - Makefile | 5 +++ pyexcel_io/book.py | 2 +- pyexcel_io/database/importers/django.py | 2 +- pyexcel_io/database/importers/sqlalchemy.py | 2 +- pyexcel_io/io.py | 6 ++-- pyexcel_io/manager.py | 3 +- pyexcel_io/plugins.py | 6 ++-- pyexcel_io/readers/csvr.py | 11 +++---- pyexcel_io/readers/csvz.py | 5 ++- pyexcel_io/readers/tsvz.py | 1 - pyexcel_io/service.py | 2 +- pyexcel_io/sheet.py | 4 +-- pyexcel_io/utils.py | 1 - pyexcel_io/writers/csvw.py | 4 +-- pyexcel_io/writers/csvz.py | 7 ++--- pyexcel_io/writers/tsvz.py | 1 - setup.py | 4 ++- tests/requirements.txt | 4 +++ tests/test_base.py | 6 ++-- tests/test_book.py | 2 +- tests/test_csv_book.py | 9 +++--- tests/test_django_book.py | 16 +++++----- tests/test_filter.py | 4 +-- tests/test_io.py | 11 +++---- tests/test_issues.py | 4 ++- tests/test_new_csv_book.py | 9 +++--- tests/test_new_csvz_book.py | 21 +++++++------ tests/test_pyexcel_integration.py | 3 +- tests/test_renderer.py | 1 + tests/test_service.py | 9 ++++-- tests/test_sheet.py | 4 +-- tests/test_sql_book.py | 34 ++++++++++++--------- 34 files changed, 110 insertions(+), 95 deletions(-) diff --git a/.moban.d/docs/source/conf.py b/.moban.d/docs/source/conf.py index 74678d7..ea8c266 100644 --- a/.moban.d/docs/source/conf.py +++ b/.moban.d/docs/source/conf.py @@ -12,4 +12,3 @@ def setup(app): {%endblock%} - diff --git a/.moban.d/setup.py b/.moban.d/setup.py index 1e6bb8f..bf85899 100644 --- a/.moban.d/setup.py +++ b/.moban.d/setup.py @@ -6,4 +6,3 @@ {%block pyexcel_extra_classifiers%} 'Programming Language :: Python :: Implementation :: PyPy' {%endblock%}} - diff --git a/Makefile b/Makefile index 7f442b5..7fa9d72 100644 --- a/Makefile +++ b/Makefile @@ -6,3 +6,8 @@ test: document: sphinx-autogen -o docs/source/generated/ docs/source/*.rst sphinx-build -b html docs/source/ docs/build/ + +format: + isort -y $(find pyexcel_io -name "*.py"|xargs echo) $(find tests -name "*.py"|xargs echo) + black -l 79 pyexcel_io + black -l 79 tests diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index 24e9b96..21db709 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -8,7 +8,7 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.manager as manager -from pyexcel_io._compact import OrderedDict, isstream, PY2 +from pyexcel_io._compact import PY2, OrderedDict, isstream from .constants import MESSAGE_ERROR_03, MESSAGE_WRONG_IO_INSTANCE diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 8935d1b..dedc94a 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -9,10 +9,10 @@ """ import logging +import pyexcel_io.constants as constants from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetWriter from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none -import pyexcel_io.constants as constants log = logging.getLogger(__name__) diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 0f8a079..876d1b2 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -7,10 +7,10 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +import pyexcel_io.constants as constants from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetWriter from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none -import pyexcel_io.constants as constants class PyexcelSQLSkipRowException(Exception): diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 4dd0e76..982408c 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -7,12 +7,12 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from types import GeneratorType import warnings +from types import GeneratorType -from pyexcel_io._compact import isstream, PY2 -from pyexcel_io.plugins import READERS, WRITERS import pyexcel_io.constants as constants +from pyexcel_io.plugins import READERS, WRITERS +from pyexcel_io._compact import PY2, isstream def iget_data(afile, file_type=None, **keywords): diff --git a/pyexcel_io/manager.py b/pyexcel_io/manager.py index 03c4586..fe3a23a 100644 --- a/pyexcel_io/manager.py +++ b/pyexcel_io/manager.py @@ -7,8 +7,7 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io._compact import StringIO, BytesIO - +from pyexcel_io._compact import BytesIO, StringIO MIME_TYPES = {} FILE_TYPES = () diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 9674929..94f4888 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -8,14 +8,12 @@ :license: New BSD License, see LICENSE for more details """ from lml.loader import scan_plugins -from lml.plugin import PluginManager -from lml.plugin import PluginInfoChain, PluginInfo +from lml.plugin import PluginInfo, PluginManager, PluginInfoChain import pyexcel_io.utils as ioutils import pyexcel_io.manager as manager -import pyexcel_io.exceptions as exceptions import pyexcel_io.constants as constants - +import pyexcel_io.exceptions as exceptions ERROR_MESSAGE_FORMATTER = "one of these plugins for %s data in '%s': %s" UPGRADE_MESSAGE = "Please upgrade the plugin '%s' according to \ diff --git a/pyexcel_io/readers/csvr.py b/pyexcel_io/readers/csvr.py index 7d168ec..50eedef 100644 --- a/pyexcel_io/readers/csvr.py +++ b/pyexcel_io/readers/csvr.py @@ -7,18 +7,17 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -import re import os +import re import csv import glob import codecs -from pyexcel_io.book import BookReader -from pyexcel_io.sheet import SheetReader, NamedContent +import pyexcel_io.service as service import pyexcel_io._compact as compact import pyexcel_io.constants as constants -import pyexcel_io.service as service - +from pyexcel_io.book import BookReader +from pyexcel_io.sheet import SheetReader, NamedContent DEFAULT_SEPARATOR = "__" DEFAULT_SHEET_SEPARATOR_FORMATTER = "---%s---" % constants.DEFAULT_NAME + "%s" @@ -84,7 +83,7 @@ class CSVMemoryMapIterator(compact.Iterator): if bom_header == BOM_BIG_ENDIAN: self.__endian = BIG_ENDIAN elif self.__endian == LITTLE_ENDIAN: - line = line[self.__zeros_left_in_2_row :] # flake8: noqa + line = line[self.__zeros_left_in_2_row :] # noqa: E203 if self.__endian == LITTLE_ENDIAN: line = line.rstrip() line = line.decode(self.__encoding) diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index bfb1956..24410b1 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -9,11 +9,10 @@ """ import zipfile -from pyexcel_io._compact import StringIO, PY2 from pyexcel_io.book import BookReader +from pyexcel_io._compact import PY2, StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ - -from .csvr import CSVinMemoryReader, NamedContent +from .csvr import NamedContent, CSVinMemoryReader class CSVZipBookReader(BookReader): diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py index 9dc7d6d..c7b7310 100644 --- a/pyexcel_io/readers/tsvz.py +++ b/pyexcel_io/readers/tsvz.py @@ -8,7 +8,6 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT - from .csvz import CSVZipBookReader diff --git a/pyexcel_io/service.py b/pyexcel_io/service.py index 9544657..493d26e 100644 --- a/pyexcel_io/service.py +++ b/pyexcel_io/service.py @@ -127,7 +127,7 @@ def time_value(value): """convert to time value accroding the specification""" import re - results = re.match("PT(\d+)H(\d+)M(\d+)S", value) + results = re.match(r"PT(\d+)H(\d+)M(\d+)S", value) if results and len(results.groups()) == 3: hour = int(results.group(1)) minute = int(results.group(2)) diff --git a/pyexcel_io/sheet.py b/pyexcel_io/sheet.py index 5d2abfb..3134b8e 100644 --- a/pyexcel_io/sheet.py +++ b/pyexcel_io/sheet.py @@ -7,9 +7,9 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io._compact import irange -from pyexcel_io.utils import _index_filter import pyexcel_io.constants as constants +from pyexcel_io.utils import _index_filter +from pyexcel_io._compact import irange class NamedContent(object): diff --git a/pyexcel_io/utils.py b/pyexcel_io/utils.py index 1413786..2207e7f 100644 --- a/pyexcel_io/utils.py +++ b/pyexcel_io/utils.py @@ -9,7 +9,6 @@ """ import pyexcel_io.constants as constants - XLS_PLUGIN = "pyexcel-xls" XLSX_PLUGIN = "pyexcel-xlsx" ODS_PLUGIN = "pyexcel-ods" diff --git a/pyexcel_io/writers/csvw.py b/pyexcel_io/writers/csvw.py index f9cd7bb..910ecfd 100644 --- a/pyexcel_io/writers/csvw.py +++ b/pyexcel_io/writers/csvw.py @@ -10,10 +10,10 @@ import csv import codecs -from pyexcel_io.book import BookWriter -from pyexcel_io.sheet import SheetWriter import pyexcel_io._compact as compact import pyexcel_io.constants as constants +from pyexcel_io.book import BookWriter +from pyexcel_io.sheet import SheetWriter class UnicodeWriter(object): diff --git a/pyexcel_io/writers/csvz.py b/pyexcel_io/writers/csvz.py index 2eb3237..76d291b 100644 --- a/pyexcel_io/writers/csvz.py +++ b/pyexcel_io/writers/csvz.py @@ -9,11 +9,10 @@ """ import zipfile -from pyexcel_io._compact import StringIO, PY2 from pyexcel_io.book import BookWriter -from pyexcel_io.constants import DEFAULT_SHEET_NAME, FILE_FORMAT_CSVZ - -from .csvw import CSVSheetWriter, UnicodeWriter +from pyexcel_io._compact import PY2, StringIO +from pyexcel_io.constants import FILE_FORMAT_CSVZ, DEFAULT_SHEET_NAME +from .csvw import UnicodeWriter, CSVSheetWriter class CSVZipSheetWriter(CSVSheetWriter): diff --git a/pyexcel_io/writers/tsvz.py b/pyexcel_io/writers/tsvz.py index b12814d..255730a 100644 --- a/pyexcel_io/writers/tsvz.py +++ b/pyexcel_io/writers/tsvz.py @@ -8,7 +8,6 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT - from .csvz import CSVZipBookWriter diff --git a/setup.py b/setup.py index 68c03f6..4cfb907 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,9 @@ import os import sys import codecs from shutil import rmtree -from setuptools import setup, find_packages, Command + +from setuptools import Command, setup, find_packages + PY2 = sys.version_info[0] == 2 PY26 = PY2 and sys.version_info[1] < 7 diff --git a/tests/requirements.txt b/tests/requirements.txt index 796e17f..29ad7b5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,3 +6,7 @@ flake8 SQLAlchemy pyexcel>=0.2.0 pyexcel-xls>=0.1.0 +moban +black;python_version>="3.6" +isort;python_version>="3.6" + diff --git a/tests/test_base.py b/tests/test_base.py index 351e7f0..a7c890e 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,7 +1,7 @@ -from pyexcel_io.sheet import SheetReader, SheetWriter, NamedContent -from pyexcel_io.book import BookWriter -from pyexcel_io.utils import is_empty_array from nose.tools import raises +from pyexcel_io.book import BookWriter +from pyexcel_io.sheet import SheetReader, SheetWriter, NamedContent +from pyexcel_io.utils import is_empty_array @raises(NotImplementedError) diff --git a/tests/test_book.py b/tests/test_book.py index 226c6ca..0e98e57 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -1,5 +1,5 @@ from nose.tools import raises -from pyexcel_io.book import RWInterface, BookReader, BookWriter +from pyexcel_io.book import BookReader, BookWriter, RWInterface @raises(NotImplementedError) diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index 0c55869..c530f7b 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -2,18 +2,19 @@ # -*- coding: utf-8 -*- import os -from unittest import TestCase from textwrap import dedent -from nose.tools import raises, eq_ +from unittest import TestCase + import pyexcel_io.manager as manager +from nose.tools import eq_, raises from pyexcel_io.sheet import NamedContent +from pyexcel_io._compact import PY2, BytesIO, StringIO from pyexcel_io.readers.csvr import ( - CSVSheetReader, CSVFileReader, + CSVSheetReader, CSVinMemoryReader, ) from pyexcel_io.writers.csvw import CSVFileWriter, CSVMemoryWriter -from pyexcel_io._compact import BytesIO, PY2, StringIO class TestReaders(TestCase): diff --git a/tests/test_django_book.py b/tests/test_django_book.py index 17fca83..95239df 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -1,20 +1,20 @@ -from nose.tools import raises, eq_ +from nose.tools import eq_, raises from pyexcel_io import save_data from pyexcel_io._compact import OrderedDict from pyexcel_io.constants import DB_DJANGO from pyexcel_io.database.common import ( - DjangoModelImporter, - DjangoModelImportAdapter, DjangoModelExporter, + DjangoModelImporter, DjangoModelExportAdapter, -) -from pyexcel_io.database.importers.django import ( - DjangoModelWriter, - DjangoBookWriter, + DjangoModelImportAdapter, ) from pyexcel_io.database.exporters.django import ( - DjangoModelReader, DjangoBookReader, + DjangoModelReader, +) +from pyexcel_io.database.importers.django import ( + DjangoBookWriter, + DjangoModelWriter, ) diff --git a/tests/test_filter.py b/tests/test_filter.py index fd7bcf8..3af8eac 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -1,9 +1,9 @@ import os +import pyexcel_io.constants as constants +from nose.tools import eq_ from pyexcel_io import get_data, save_data from pyexcel_io.utils import _index_filter -from nose.tools import eq_ -import pyexcel_io.constants as constants def test_index_filter(): diff --git a/tests/test_io.py b/tests/test_io.py index 8670c18..f373a4e 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,16 +1,15 @@ import os import sys import types +from zipfile import BadZipfile from unittest import TestCase + import pyexcel_io.manager as manager import pyexcel_io.exceptions as exceptions -from pyexcel_io._compact import StringIO, BytesIO, is_string -from pyexcel_io._compact import OrderedDict -from pyexcel_io import save_data, get_data, iget_data +from nose.tools import eq_, raises +from pyexcel_io import get_data, iget_data, save_data from pyexcel_io.io import load_data, get_writer -from nose.tools import raises, eq_ -from zipfile import BadZipfile - +from pyexcel_io._compact import BytesIO, StringIO, OrderedDict, is_string PY2 = sys.version_info[0] == 2 diff --git a/tests/test_issues.py b/tests/test_issues.py index 81704f8..7a436d2 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -2,11 +2,13 @@ # -*- coding: utf-8 -*- import os + +import pyexcel as p + from nose import SkipTest from nose.tools import eq_ from pyexcel_io import get_data, save_data from pyexcel_io._compact import PY26 -import pyexcel as p IN_TRAVIS = "TRAVIS" in os.environ diff --git a/tests/test_new_csv_book.py b/tests/test_new_csv_book.py index e9fa908..857f4ee 100644 --- a/tests/test_new_csv_book.py +++ b/tests/test_new_csv_book.py @@ -1,13 +1,14 @@ import os -from unittest import TestCase from textwrap import dedent -from nose.tools import raises +from unittest import TestCase + import pyexcel_io.manager as manager +from nose.tools import raises from pyexcel_io._compact import OrderedDict -from pyexcel_io.readers.csvr import CSVBookReader from pyexcel_io.readers.tsv import TSVBookReader -from pyexcel_io.writers.csvw import CSVBookWriter from pyexcel_io.writers.tsv import TSVBookWriter +from pyexcel_io.readers.csvr import CSVBookReader +from pyexcel_io.writers.csvw import CSVBookWriter class TestCSVReaders(TestCase): diff --git a/tests/test_new_csvz_book.py b/tests/test_new_csvz_book.py index fb7892a..fdf8620 100644 --- a/tests/test_new_csvz_book.py +++ b/tests/test_new_csvz_book.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- import os -from unittest import TestCase -from pyexcel_io._compact import OrderedDict -from pyexcel_io import save_data -import pyexcel_io.manager as manager -from pyexcel_io.readers.csvz import CSVZipBookReader -from pyexcel_io.writers.csvz import CSVZipBookWriter -from pyexcel_io.readers.tsvz import TSVZipBookReader -from pyexcel_io.writers.tsvz import TSVZipBookWriter -import zipfile -from nose.tools import raises import sys +import zipfile +from unittest import TestCase + +import pyexcel_io.manager as manager +from nose.tools import raises +from pyexcel_io import save_data +from pyexcel_io._compact import OrderedDict +from pyexcel_io.readers.csvz import CSVZipBookReader +from pyexcel_io.readers.tsvz import TSVZipBookReader +from pyexcel_io.writers.csvz import CSVZipBookWriter +from pyexcel_io.writers.tsvz import TSVZipBookWriter PY2 = sys.version_info[0] == 2 diff --git a/tests/test_pyexcel_integration.py b/tests/test_pyexcel_integration.py index a13a80f..e198c74 100644 --- a/tests/test_pyexcel_integration.py +++ b/tests/test_pyexcel_integration.py @@ -1,9 +1,10 @@ import os import datetime -from unittest import TestCase from textwrap import dedent +from unittest import TestCase import pyexcel as pe + from pyexcel_io._compact import text_type diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 38caf8a..5be2174 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -1,4 +1,5 @@ import os + from nose.tools import eq_ from pyexcel_io import get_data, save_data diff --git a/tests/test_service.py b/tests/test_service.py index feb652c..552fb10 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,7 +1,10 @@ from nose.tools import eq_, raises -from pyexcel_io.service import date_value, time_value -from pyexcel_io.service import detect_int_value -from pyexcel_io.service import detect_float_value +from pyexcel_io.service import ( + date_value, + time_value, + detect_int_value, + detect_float_value, +) def test_date_util_parse(): diff --git a/tests/test_sheet.py b/tests/test_sheet.py index 3e3b7ac..789317a 100644 --- a/tests/test_sheet.py +++ b/tests/test_sheet.py @@ -1,6 +1,6 @@ -from nose.tools import eq_ -from pyexcel_io.sheet import SheetWriter, SheetReader import pyexcel_io.constants as constants +from nose.tools import eq_ +from pyexcel_io.sheet import SheetReader, SheetWriter class MyWriter(SheetWriter): diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 6691ae6..6927056 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -1,32 +1,38 @@ import sys import json -from sqlalchemy import create_engine -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import Column, Integer, String -from sqlalchemy import Float, Date, DateTime, ForeignKey -from sqlalchemy.orm import sessionmaker import datetime +import platform + +from nose.tools import eq_, raises +from sqlalchemy import ( + Date, + Float, + Column, + String, + Integer, + DateTime, + ForeignKey, + create_engine, +) +from sqlalchemy.orm import backref, relationship, sessionmaker from pyexcel_io._compact import OrderedDict from pyexcel_io.database.common import ( SQLTableExporter, - SQLTableExportAdapter, SQLTableImporter, + SQLTableExportAdapter, SQLTableImportAdapter, ) +from sqlalchemy.ext.declarative import declarative_base +from pyexcel_io.database.querysets import QuerysetsReader from pyexcel_io.database.exporters.sqlalchemy import ( - SQLTableReader, SQLBookReader, + SQLTableReader, ) from pyexcel_io.database.importers.sqlalchemy import ( - PyexcelSQLSkipRowException, - SQLTableWriter, SQLBookWriter, + SQLTableWriter, + PyexcelSQLSkipRowException, ) -from pyexcel_io.database.querysets import QuerysetsReader -from sqlalchemy.orm import relationship, backref -from nose.tools import raises, eq_ -import platform - PY3 = sys.version_info[0] == 3 PY36 = PY3 and sys.version_info[1] == 6 From fc0e6088f3a12404234a11f3974790f39b01699e Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 9 Nov 2018 21:02:13 +0000 Subject: [PATCH 002/143] :sparkles: use moban 0.3.3 --- .moban.d/tests/requirements.txt | 3 +++ .moban.yml | 11 ++++++++--- docs/source/conf.py | 9 ++------- setup.py | 2 +- tests/requirements.txt | 1 - 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.moban.d/tests/requirements.txt b/.moban.d/tests/requirements.txt index ea7c0e3..f5f7679 100644 --- a/.moban.d/tests/requirements.txt +++ b/.moban.d/tests/requirements.txt @@ -3,4 +3,7 @@ SQLAlchemy pyexcel>=0.2.0 pyexcel-xls>=0.1.0 +moban +black;python_version>="3.6" +isort;python_version>="3.6" {%endblock%} diff --git a/.moban.yml b/.moban.yml index d87341c..afc7f5b 100644 --- a/.moban.yml +++ b/.moban.yml @@ -1,8 +1,13 @@ +requires: + - type: git + url: https://github.com/moremoban/pypi-mobans + submodule: true + - https://github.com/pyexcel/pyexcel-mobans configuration: - configuration_dir: "commons/config" + configuration_dir: "pyexcel-mobans:config" template_dir: - - "commons/templates" - - "setupmobans/templates" + - "pyexcel-mobans:templates" + - "pypi-mobans:templates" - ".moban.d" configuration: pyexcel-io.yml targets: diff --git a/docs/source/conf.py b/docs/source/conf.py index 7a7c49b..5728565 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,12 +43,7 @@ release = u'0.5.9.1' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.viewcode', -] +extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode',] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -75,7 +70,7 @@ language = 'en' exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = None # -- Options for HTML output ------------------------------------------------- diff --git a/setup.py b/setup.py index 4cfb907..10513ff 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ URL = 'https://github.com/pyexcel/pyexcel-io' DOWNLOAD_URL = '%s/archive/0.5.9.1.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ + 'python', 'API', 'tsv', 'tsvz', @@ -31,7 +32,6 @@ KEYWORDS = [ 'csvz', 'django', 'sqlalchemy', - 'python', ] CLASSIFIERS = [ diff --git a/tests/requirements.txt b/tests/requirements.txt index 29ad7b5..787faf3 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,4 +9,3 @@ pyexcel-xls>=0.1.0 moban black;python_version>="3.6" isort;python_version>="3.6" - From 43a5a56227128fbdcc39b6a2932571aac0f1adbd Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Nov 2018 22:32:18 +0000 Subject: [PATCH 003/143] :hammer: code refactoring on v0.6.0 --- docs/source/conf.py | 2 +- docs/source/index.rst | 1 + pyexcel-io.yml | 4 ++-- pyexcel_io/io.py | 20 ++------------------ rnd_requirements.txt | 2 +- setup.py | 2 +- 6 files changed, 8 insertions(+), 23 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 5728565..fcd5002 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -31,7 +31,7 @@ author = u'C.W.' # The short X.Y version version = u'0.5.9.1' # The full version, including alpha/beta/rc tags -release = u'0.5.9.1' +release = u'0.6.0' # -- General configuration --------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 7638344..5bc9765 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License +:Development: |release| :Released: |version| :Generated: |today| diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 2dc8a6c..f07339f 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -1,8 +1,8 @@ overrides: "pyexcel.yaml" name: "pyexcel-io" nick_name: io -version: 0.5.9.1 -current_version: 0.5.9.1 +version: 0.6.0 +current_version: 0.6.0 release: 0.5.9.1 dependencies: - ordereddict;python_version<"2.7" diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 982408c..6d68f59 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -119,29 +119,13 @@ def save_data(afile, data, file_type=None, **keywords): if no_file_type: file_type = constants.FILE_FORMAT_CSV - store_data( - afile, - to_store, - file_type=file_type, - single_sheet_in_book=single_sheet_in_book, - **keywords - ) - - -def store_data(afile, data, file_type=None, **keywords): - """Non public function to store data to afile - - :param filename: actual file name, a file stream or actual content - :param data: the data to be written - :param file_type: used only when filename is not a physial file name - :param keywords: any other parameters - """ if isstream(afile): keywords.update(dict(file_stream=afile, file_type=file_type)) else: keywords.update(dict(file_name=afile, file_type=file_type)) + keywords['single_sheet_in_book'] = single_sheet_in_book with get_writer(**keywords) as writer: - writer.write(data) + writer.write(to_store) def load_data( diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 5d515e1..676fcd7 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,5 +1,5 @@ . https://github.com/chfw/lml/archive/master.zip https://github.com/pyexcel/pyexcel/archive/master.zip -https://github.com/pyexcel/pyexcel-xls/archive/master.zip +https://github.com/pyexcel/pyexcel-xls/archive/dev.zip diff --git a/setup.py b/setup.py index 10513ff..b106590 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ PY26 = PY2 and sys.version_info[1] < 7 NAME = 'pyexcel-io' AUTHOR = 'C.W.' -VERSION = '0.5.9.1' +VERSION = '0.6.0' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'New BSD' DESCRIPTION = ( From 33bd35d9b32567ef05522ccaabe6a1aa30bc5d4a Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 9 Feb 2019 22:27:44 +0000 Subject: [PATCH 004/143] :books: prepare for new release --- .moban.yml | 2 +- CHANGELOG.rst | 11 +++- LICENSE | 2 +- changelog.yml | 6 +++ docs/source/conf.py | 116 +++++------------------------------------- docs/source/index.rst | 1 - pyexcel-io.yml | 9 ++-- setup.py | 24 +++++++-- 8 files changed, 55 insertions(+), 116 deletions(-) diff --git a/.moban.yml b/.moban.yml index afc7f5b..01071fa 100644 --- a/.moban.yml +++ b/.moban.yml @@ -11,7 +11,7 @@ configuration: - ".moban.d" configuration: pyexcel-io.yml targets: - - "docs/source/conf.py": "docs/source/conf.py" + - "docs/source/conf.py": "docs/conf.py_t" - setup.py: setup.py - .travis.yml: travis.yml.jj2 - requirements.txt: requirements.txt.jj2 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 410225f..dab9013 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,16 @@ Change log ================================================================================ -0.5.10 - 3.12.2018 +0.5.12 - 9.02.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#60 `_: include tests in + tar ball + +0.5.11 - 3.12.2018 -------------------------------------------------------------------------------- updated diff --git a/LICENSE b/LICENSE index eb6ae75..2bc69b9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2018 by Onni Software Ltd. and its contributors +Copyright (c) by Onni Software Ltd. and its contributors All rights reserved. Redistribution and use in source and binary forms of the software as well diff --git a/changelog.yml b/changelog.yml index 63cbc90..ee31a80 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`#60`: include tests in tar ball' + version: 0.5.12 + date: 9.02.2019 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index fa39c2f..586c5b4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,15 +1,7 @@ -# -*- coding: utf-8 -*- -DESCRIPTION = ( - 'A python library to read and write structured data in csv, zipped csv ' + - 'format and to/from databases' + - '' -) -# -*- coding: utf-8 -*- -# # Configuration file for the Sphinx documentation builder. # -# This file does only contain a selection of the most common options. For a -# full list see the documentation: +# This file only contains a selection of the most common options. For a full +# list see the documentation: # http://www.sphinx-doc.org/en/master/config # -- Path setup -------------------------------------------------------------- @@ -24,54 +16,47 @@ DESCRIPTION = ( # -- Project information ----------------------------------------------------- -project = u'pyexcel-io' -copyright = u'2015-2018 Onni Software Ltd.' -author = u'C.W.' +project = 'pyexcel-io' +copyright = 'copyright 2015-2019 Onni Software Ltd.' +author = 'C.W.' # The short X.Y version -version = u'0.5.11' +version = '0.5.12' # The full version, including alpha/beta/rc tags -release = u'0.6.0' +release = '0.5.12' # -- General configuration --------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode',] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ['templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = '' # The master toctree document. -master_doc = 'index' +master_doc = '' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = 'en' +language = '' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - # -- Options for HTML output ------------------------------------------------- @@ -89,7 +74,7 @@ html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ['static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -102,63 +87,6 @@ html_static_path = ['_static'] # html_sidebars = {} -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = 'pyexcel-iodoc' - - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'pyexcel-io.tex', u'pyexcel-io Documentation', - u'Onni Software Ltd.', 'manual'), -] - - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pyexcel-io', u'pyexcel-io Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'pyexcel-io', u'pyexcel-io Documentation', - author, 'pyexcel-io', 'One line description of project.', - 'Miscellaneous'), -] - - # -- Options for Epub output ------------------------------------------------- # Bibliographic Dublin Core info. @@ -181,23 +109,3 @@ epub_exclude_files = ['search.html'] # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} -# TODO: html_theme not configurable upstream -html_theme = 'default' - - -def setup(app): - app.add_stylesheet('theme_overrides.css') - - - -# TODO: DESCRIPTION not configurable upstream -texinfo_documents = [ - ('index', 'pyexcel-io', - 'pyexcel-io Documentation', - 'Onni Software Ltd.', 'pyexcel-io', - DESCRIPTION, - 'Miscellaneous'), -] -intersphinx_mapping.update({ - 'pyexcel': ('http://pyexcel.readthedocs.io/en/latest/', None), -}) diff --git a/docs/source/index.rst b/docs/source/index.rst index 5bc9765..7638344 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,6 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License -:Development: |release| :Released: |version| :Generated: |today| diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 2e5efb1..5f62167 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -1,9 +1,10 @@ overrides: "pyexcel.yaml" -name: "pyexcel-io" +project: "pyexcel-io" +name: pyexcel-io nick_name: io -version: 0.6.0 -current_version: 0.6.0 -release: 0.5.11 +version: 0.5.12 +current_version: 0.5.12 +release: 0.5.12 dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 diff --git a/setup.py b/setup.py index 6153faa..96e7861 100644 --- a/setup.py +++ b/setup.py @@ -4,16 +4,32 @@ import os import sys import codecs +import locale +import platform from shutil import rmtree from setuptools import Command, setup, find_packages PY2 = sys.version_info[0] == 2 PY26 = PY2 and sys.version_info[1] < 7 +PY33 = sys.version_info < (3, 4) + +# Work around mbcs bug in distutils. +# http://bugs.python.org/issue10945 +# This work around is only if a project supports Python < 3.4 + +# Work around for locale not being set +try: + lc = locale.getlocale() + pf = platform.system() + if pf != 'Windows' and lc == (None, None): + locale.setlocale(locale.LC_ALL, 'C.UTF-8') +except (ValueError, UnicodeError, locale.Error): + locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') NAME = 'pyexcel-io' AUTHOR = 'C.W.' -VERSION = '0.6.0' +VERSION = '0.5.12' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'New BSD' DESCRIPTION = ( @@ -21,7 +37,7 @@ DESCRIPTION = ( 'format and to/from databases' ) URL = 'https://github.com/pyexcel/pyexcel-io' -DOWNLOAD_URL = '%s/archive/0.5.11.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.5.12.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -65,8 +81,8 @@ EXTRAS_REQUIRE = { # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs pyexcel-io v0.5.11 ' + - "Find 0.5.11 in changelog for more details") +GS_COMMAND = ('gs pyexcel-io v0.5.12 ' + + "Find 0.5.12 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( From 8542da1900af65370a073b466b097b809db6f7bd Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 9 Feb 2019 22:28:35 +0000 Subject: [PATCH 005/143] :bug: fix incompatibility with sphinx conf --- docs/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 586c5b4..70dc6b6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,14 +43,14 @@ templates_path = ['templates'] source_suffix = '' # The master toctree document. -master_doc = '' +master_doc = 'index' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = '' +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From d8d076b85d1c841721201c37351e713106a49f94 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 9 Feb 2019 22:38:03 +0000 Subject: [PATCH 006/143] :newspaper: include more test content in tar ball. resolve #60 --- MANIFEST.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index c2e4b1b..1e74856 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,7 @@ include README.rst include LICENSE include CHANGELOG.rst +recursive-include tests * +include docs/source/* +include Makefile +include test.sh From e6010196e9d60a82dfb1baf226dc1b76de637a88 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 9 Feb 2019 22:52:55 +0000 Subject: [PATCH 007/143] :sparkles: enable python setup.py test, though it is partial. fix #61 --- .moban.yml | 1 - setup.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.moban.yml b/.moban.yml index 01071fa..a062f67 100644 --- a/.moban.yml +++ b/.moban.yml @@ -17,7 +17,6 @@ targets: - requirements.txt: requirements.txt.jj2 - "tests/requirements.txt": "tests/requirements.txt" - LICENSE: NEW_BSD_LICENSE.jj2 - - MANIFEST.in: MANIFEST.in.jj2 - test.sh: test.script.jj2 - test.bat: test.script.jj2 - README.rst: README.rst diff --git a/setup.py b/setup.py index 96e7861..7746b7d 100644 --- a/setup.py +++ b/setup.py @@ -191,6 +191,7 @@ def filter_out_test_code(file_handle): if __name__ == '__main__': setup( + test_suite="tests", name=NAME, author=AUTHOR, version=VERSION, From 0bff65d187c1df0505f583f6f23246310fc393b1 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 9 Feb 2019 22:53:39 +0000 Subject: [PATCH 008/143] :book: update changelog --- CHANGELOG.rst | 2 ++ changelog.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dab9013..962bccb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ updated #. `#60 `_: include tests in tar ball +#. `#61 `_: enable python + setup.py test 0.5.11 - 3.12.2018 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index ee31a80..c5302b9 100644 --- a/changelog.yml +++ b/changelog.yml @@ -5,6 +5,7 @@ releases: - action: updated details: - '`#60`: include tests in tar ball' + - '`#61`: enable python setup.py test' version: 0.5.12 date: 9.02.2019 - changes: From 588ad8931ac3638738c6c0fff8ad2aea615cd171 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 20 Feb 2019 18:32:57 +0000 Subject: [PATCH 009/143] :newspaper: make python2 setup.py test happy. fix #65 --- tests/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..4f03500 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# needed for Python 2.7, python setup.py test to discover tests directory From e0db1844d9331062aee91797b3ba40b65d4b16c3 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 21 Feb 2019 07:29:44 +0000 Subject: [PATCH 010/143] :egg: :ferris_wheel: release 0.5.13 --- .travis.yml | 8 +++++--- CHANGELOG.rst | 9 +++++++++ changelog.yml | 6 ++++++ docs/source/conf.py | 45 +++------------------------------------------ pyexcel-io.yml | 6 +++--- setup.py | 8 ++++---- 6 files changed, 30 insertions(+), 52 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8295c53..ee3a11f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,15 @@ sudo: false +dist: xenial language: python notifications: email: false python: - - pypy-5.3.1 - - 3.7-dev + - &pypy2 pypy2.7-6.0 + - &pypy3 pypy3.5-6.0 + - 3.8-dev + - 3.7 - 3.6 - 3.5 - - 3.4 - 2.7 before_install: - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 962bccb..74feb60 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.5.13 - 21.02.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#65 `_: add + tests/__init__.py because python2.7 setup.py test needs it + 0.5.12 - 9.02.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index c5302b9..2563558 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`#65`: add tests/__init__.py because python2.7 setup.py test needs it' + version: 0.5.13 + date: 21.02.2019 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index 70dc6b6..91a7f50 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,12 +19,10 @@ project = 'pyexcel-io' copyright = 'copyright 2015-2019 Onni Software Ltd.' author = 'C.W.' - # The short X.Y version -version = '0.5.12' +version = '0.5.13' # The full version, including alpha/beta/rc tags -release = '0.5.12' - +release = '0.5.13' # -- General configuration --------------------------------------------------- @@ -42,9 +40,6 @@ templates_path = ['templates'] # source_suffix = ['.rst', '.md'] source_suffix = '' -# The master toctree document. -master_doc = 'index' - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # @@ -65,47 +60,13 @@ exclude_patterns = [] # html_theme = 'alabaster' -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['static'] -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - - -# -- Options for Epub output ------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] - # -- Extension configuration ------------------------------------------------- # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {'https://docs.python.org/': None} \ No newline at end of file diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 5f62167..bbb9cf7 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,9 +2,9 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.5.12 -current_version: 0.5.12 -release: 0.5.12 +version: 0.5.13 +current_version: 0.5.13 +release: 0.5.13 dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 diff --git a/setup.py b/setup.py index 7746b7d..6d1e8bc 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = 'pyexcel-io' AUTHOR = 'C.W.' -VERSION = '0.5.12' +VERSION = '0.5.13' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'New BSD' DESCRIPTION = ( @@ -37,7 +37,7 @@ DESCRIPTION = ( 'format and to/from databases' ) URL = 'https://github.com/pyexcel/pyexcel-io' -DOWNLOAD_URL = '%s/archive/0.5.12.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.5.13.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -81,8 +81,8 @@ EXTRAS_REQUIRE = { # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs pyexcel-io v0.5.12 ' + - "Find 0.5.12 in changelog for more details") +GS_COMMAND = ('gs pyexcel-io v0.5.13 ' + + "Find 0.5.13 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( From 03641589be71f18bd016de3c4b58ea22dacc952c Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 21 Feb 2019 08:17:26 +0000 Subject: [PATCH 011/143] :egg: :ferris_wheel: release 0.5.13 --- .moban.d/docs/source/index.rst | 3 +- .travis.yml | 8 ++- CHANGELOG.rst | 22 +++++- LICENSE | 2 +- MANIFEST.in | 2 + README.rst | 1 + changelog.yml | 15 +++- docs/source/conf.py | 125 ++------------------------------- pyexcel-io.yml | 10 +-- setup.py | 25 +++++-- 10 files changed, 80 insertions(+), 133 deletions(-) diff --git a/.moban.d/docs/source/index.rst b/.moban.d/docs/source/index.rst index 8b9c737..bd24745 100644 --- a/.moban.d/docs/source/index.rst +++ b/.moban.d/docs/source/index.rst @@ -57,7 +57,8 @@ get_data(.., library='pyexcel-ods') ============= ======= ======== ======= ======== ======== ======== `pyexcel-io`_ `xls`_ `xlsx`_ `ods`_ `ods3`_ `odsr`_ `xlsxw`_ ============= ======= ======== ======= ======== ======== ======== - 0.5.1 0.5.0 0.5.0 0.5.0 0.5.0 0.5.0 0.5.0 + 0.5.10+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+ + 0.5.1+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.3.0+ 0.3.0+ 0.3.0 0.3.0+ 0.3.0+ 0.3.0 0.3.0 0.2.2+ 0.2.2+ 0.2.2+ 0.2.1+ 0.2.1+ 0.0.1 diff --git a/.travis.yml b/.travis.yml index 8295c53..ee3a11f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,15 @@ sudo: false +dist: xenial language: python notifications: email: false python: - - pypy-5.3.1 - - 3.7-dev + - &pypy2 pypy2.7-6.0 + - &pypy3 pypy3.5-6.0 + - 3.8-dev + - 3.7 - 3.6 - 3.5 - - 3.4 - 2.7 before_install: - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 410225f..74feb60 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,27 @@ Change log ================================================================================ -0.5.10 - 3.12.2018 +0.5.13 - 21.02.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#65 `_: add + tests/__init__.py because python2.7 setup.py test needs it + +0.5.12 - 9.02.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#60 `_: include tests in + tar ball +#. `#61 `_: enable python + setup.py test + +0.5.11 - 3.12.2018 -------------------------------------------------------------------------------- updated diff --git a/LICENSE b/LICENSE index eb6ae75..e763169 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2018 by Onni Software Ltd. and its contributors +Copyright (c) 2015-2019 by Onni Software Ltd. and its contributors All rights reserved. Redistribution and use in source and binary forms of the software as well diff --git a/MANIFEST.in b/MANIFEST.in index c2e4b1b..b1bf562 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include README.rst include LICENSE include CHANGELOG.rst +recursive-include tests * +recursive-include docs * diff --git a/README.rst b/README.rst index 92358b7..30562fd 100644 --- a/README.rst +++ b/README.rst @@ -211,6 +211,7 @@ Acceptance criteria #. Has all code lines tested #. Passes all Travis CI builds #. Has fair amount of documentation if your change is complex +#. run 'make format' so as to confirm the pyexcel organisation's coding style #. Please update CHANGELOG.rst #. Please add yourself to CONTRIBUTORS.rst #. Agree on NEW BSD License for your contribution diff --git a/changelog.yml b/changelog.yml index bee720a..2563558 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,11 +1,24 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`#65`: add tests/__init__.py because python2.7 setup.py test needs it' + version: 0.5.13 + date: 21.02.2019 +- changes: + - action: updated + details: + - '`#60`: include tests in tar ball' + - '`#61`: enable python setup.py test' + version: 0.5.12 + date: 9.02.2019 - changes: - action: updated details: - '`#59`: Please use scan_plugins_regex, which lml 0.7 complains about' - version: 0.5.10 + version: 0.5.11 date: 3.12.2018 - changes: - action: added diff --git a/docs/source/conf.py b/docs/source/conf.py index 29f2cd4..eb3a009 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -4,12 +4,10 @@ DESCRIPTION = ( 'format and to/from databases' + '' ) -# -*- coding: utf-8 -*- -# # Configuration file for the Sphinx documentation builder. # -# This file does only contain a selection of the most common options. For a -# full list see the documentation: +# This file only contains a selection of the most common options. For a full +# list see the documentation: # http://www.sphinx-doc.org/en/master/config # -- Path setup -------------------------------------------------------------- @@ -24,22 +22,16 @@ DESCRIPTION = ( # -- Project information ----------------------------------------------------- -project = u'pyexcel-io' -copyright = u'2015-2018 Onni Software Ltd.' -author = u'C.W.' - +project = 'pyexcel-io' +copyright = 'copyright 2015-2019 Onni Software Ltd.' +author = 'Onni Software Ltd.' # The short X.Y version -version = u'0.5.11' +version = '0.5.13' # The full version, including alpha/beta/rc tags -release = u'0.5.11' - +release = '0.5.13' # -- General configuration --------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. @@ -48,15 +40,6 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext. # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # @@ -69,9 +52,6 @@ language = 'en' # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - # -- Options for HTML output ------------------------------------------------- @@ -80,102 +60,11 @@ pygments_style = None # html_theme = 'alabaster' -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - - -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = 'pyexcel-iodoc' - - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'pyexcel-io.tex', u'pyexcel-io Documentation', - u'Onni Software Ltd.', 'manual'), -] - - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pyexcel-io', u'pyexcel-io Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'pyexcel-io', u'pyexcel-io Documentation', - author, 'pyexcel-io', 'One line description of project.', - 'Miscellaneous'), -] - - -# -- Options for Epub output ------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] - # -- Extension configuration ------------------------------------------------- # -- Options for intersphinx extension --------------------------------------- diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 9465bb3..0b1907f 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -1,9 +1,11 @@ overrides: "pyexcel.yaml" -name: "pyexcel-io" +project: "pyexcel-io" +name: pyexcel-io nick_name: io -version: 0.5.11 -current_version: 0.5.11 -release: 0.5.11 +version: 0.5.13 +current_version: 0.5.13 +copyright_year: 2015-2019 +release: 0.5.13 dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 diff --git a/setup.py b/setup.py index dd5852d..6d1e8bc 100644 --- a/setup.py +++ b/setup.py @@ -4,16 +4,32 @@ import os import sys import codecs +import locale +import platform from shutil import rmtree from setuptools import Command, setup, find_packages PY2 = sys.version_info[0] == 2 PY26 = PY2 and sys.version_info[1] < 7 +PY33 = sys.version_info < (3, 4) + +# Work around mbcs bug in distutils. +# http://bugs.python.org/issue10945 +# This work around is only if a project supports Python < 3.4 + +# Work around for locale not being set +try: + lc = locale.getlocale() + pf = platform.system() + if pf != 'Windows' and lc == (None, None): + locale.setlocale(locale.LC_ALL, 'C.UTF-8') +except (ValueError, UnicodeError, locale.Error): + locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') NAME = 'pyexcel-io' AUTHOR = 'C.W.' -VERSION = '0.5.11' +VERSION = '0.5.13' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'New BSD' DESCRIPTION = ( @@ -21,7 +37,7 @@ DESCRIPTION = ( 'format and to/from databases' ) URL = 'https://github.com/pyexcel/pyexcel-io' -DOWNLOAD_URL = '%s/archive/0.5.11.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.5.13.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -65,8 +81,8 @@ EXTRAS_REQUIRE = { # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs pyexcel-io v0.5.11 ' + - "Find 0.5.11 in changelog for more details") +GS_COMMAND = ('gs pyexcel-io v0.5.13 ' + + "Find 0.5.13 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( @@ -175,6 +191,7 @@ def filter_out_test_code(file_handle): if __name__ == '__main__': setup( + test_suite="tests", name=NAME, author=AUTHOR, version=VERSION, From 230b628b3900a7d01fa0db71b694aebaabf12cb8 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 16 Mar 2019 14:28:25 +0000 Subject: [PATCH 012/143] :green_heart: try fix https://github.com/pyexcel/pyexcel-ods/issues/33 --- pyexcel_io/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexcel_io/service.py b/pyexcel_io/service.py index fc562bb..2b7710c 100644 --- a/pyexcel_io/service.py +++ b/pyexcel_io/service.py @@ -198,7 +198,7 @@ def throw_exception(value): def ods_float_value(value): - if value > constants.MAX_INTEGER: + if int(value) > int(constants.MAX_INTEGER): raise exceptions.IntegerAccuracyLossError("%s is too big" % value) return value From 29a7ffcbaab1aef8aa5eca86ded96375e0a50b26 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 16 Mar 2019 14:35:44 +0000 Subject: [PATCH 013/143] :handshake: sync with latest pyexcel-commons --- .moban.yml | 1 + .travis.yml | 14 ++++++++++++++ CHANGELOG.rst | 9 +++++++++ Makefile | 4 ++++ README.rst | 7 +++++++ changelog.yml | 6 ++++++ docs/source/conf.py | 4 ++-- docs/source/index.rst | 7 +++++++ lint.sh | 1 + pyexcel-io.yml | 6 +++--- setup.py | 8 ++++---- test.bat | 2 +- test.sh | 2 +- tests/__init__.py | 0 14 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 lint.sh create mode 100644 tests/__init__.py diff --git a/.moban.yml b/.moban.yml index f773b2b..529a249 100644 --- a/.moban.yml +++ b/.moban.yml @@ -24,3 +24,4 @@ targets: - output: CHANGELOG.rst configuration: changelog.yml template: CHANGELOG.rst.jj2 + - lint.sh: lint.script.jj2 diff --git a/.travis.yml b/.travis.yml index ee3a11f..c2c6b83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,20 @@ python: - 3.6 - 3.5 - 2.7 + +stages: + - test + - lint +jobs: + include: + - stage: lint + python: 3.6 + script: make lint + +stage: test + +script: make test + before_install: - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1e4f69c..56917c6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.5.15 - 16.03.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `pyexcel-ods#33 `_: fix + integer comparision error on i586 + 0.5.14 - 21.02.2019 -------------------------------------------------------------------------------- diff --git a/Makefile b/Makefile index 7f442b5..7fe459a 100644 --- a/Makefile +++ b/Makefile @@ -6,3 +6,7 @@ test: document: sphinx-autogen -o docs/source/generated/ docs/source/*.rst sphinx-build -b html docs/source/ docs/build/ + +lint: + bash lint.sh + diff --git a/README.rst b/README.rst index 30562fd..62ae98b 100644 --- a/README.rst +++ b/README.rst @@ -78,8 +78,11 @@ sqlalchemy supported databases. Its supported file formats are extended to cover ======================== ======================= ================= ================== `pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ Python 2 and 3 `pyexcel-xlsxr`_ xlsx(read only) lxml same as above + `pyexcel-xlsbr`_ xlsx(read only) pyxlsb same as above `pyexcel-odsr`_ read only for ods, fods lxml same as above + `pyexcel-odsw`_ write only for ods loxun same as above `pyexcel-htmlr`_ html(read only) lxml,html5lib same as above + `pyexcel-pdfr`_ pdf(read only) pdftables Python 2 only. ======================== ======================= ================= ================== @@ -89,8 +92,12 @@ sqlalchemy supported databases. Its supported file formats are extended to cover .. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods .. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3 .. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr +.. _pyexcel-odsw: https://github.com/pyexcel/pyexcel-odsw +.. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr + .. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw .. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr +.. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr .. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr .. _xlrd: https://github.com/python-excel/xlrd diff --git a/changelog.yml b/changelog.yml index a916068..ed0bdce 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`pyexcel-ods#33`: fix integer comparision error on i586' + version: 0.5.15 + date: 16.03.2019 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index 7741c61..bed6d04 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,9 +26,9 @@ project = 'pyexcel-io' copyright = 'copyright 2015-2019 Onni Software Ltd.' author = 'Onni Software Ltd.' # The short X.Y version -version = '0.5.14' +version = '0.5.15' # The full version, including alpha/beta/rc tags -release = '0.5.14' +release = '0.5.15' # -- General configuration --------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index af27e3a..f5f2be0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -85,8 +85,11 @@ For individual excel file formats, please install them as you wish: ======================== ======================= ================= ================== `pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ Python 2 and 3 `pyexcel-xlsxr`_ xlsx(read only) lxml same as above + `pyexcel-xlsbr`_ xlsx(read only) pyxlsb same as above `pyexcel-odsr`_ read only for ods, fods lxml same as above + `pyexcel-odsw`_ write only for ods loxun same as above `pyexcel-htmlr`_ html(read only) lxml,html5lib same as above + `pyexcel-pdfr`_ pdf(read only) pdftables Python 2 only. ======================== ======================= ================= ================== @@ -96,8 +99,12 @@ For individual excel file formats, please install them as you wish: .. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods .. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3 .. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr +.. _pyexcel-odsw: https://github.com/pyexcel/pyexcel-odsw +.. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr + .. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw .. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr +.. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr .. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr .. _xlrd: https://github.com/python-excel/xlrd diff --git a/lint.sh b/lint.sh new file mode 100644 index 0000000..12183c3 --- /dev/null +++ b/lint.sh @@ -0,0 +1 @@ +flake8 . --exclude=.moban.d,docs --builtins=unicode,xrange,long \ No newline at end of file diff --git a/pyexcel-io.yml b/pyexcel-io.yml index ca50643..f906d2f 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,10 +2,10 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.5.14 -current_version: 0.5.14 +version: 0.5.15 +current_version: 0.5.15 copyright_year: 2015-2019 -release: 0.5.14 +release: 0.5.15 dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 diff --git a/setup.py b/setup.py index 7c7ad1e..cf93e8d 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = 'pyexcel-io' AUTHOR = 'C.W.' -VERSION = '0.5.14' +VERSION = '0.5.15' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'New BSD' DESCRIPTION = ( @@ -37,7 +37,7 @@ DESCRIPTION = ( 'format and to/from databases' ) URL = 'https://github.com/pyexcel/pyexcel-io' -DOWNLOAD_URL = '%s/archive/0.5.14.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.5.15.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -81,8 +81,8 @@ EXTRAS_REQUIRE = { # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs pyexcel-io v0.5.14 ' + - "Find 0.5.14 in changelog for more details") +GS_COMMAND = ('gs pyexcel-io v0.5.15 ' + + "Find 0.5.15 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( diff --git a/test.bat b/test.bat index b083787..9e24a8c 100644 --- a/test.bat +++ b/test.bat @@ -1,2 +1,2 @@ pip freeze -nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io && flake8 . --exclude=.moban.d,docs --builtins=unicode,xrange,long +nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io diff --git a/test.sh b/test.sh index b083787..9e24a8c 100644 --- a/test.sh +++ b/test.sh @@ -1,2 +1,2 @@ pip freeze -nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io && flake8 . --exclude=.moban.d,docs --builtins=unicode,xrange,long +nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 4034787f46dfabf51bd8487add8b569df12234ac Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 19 Mar 2019 07:35:45 +0000 Subject: [PATCH 014/143] :bug: fix the conversion issue for long type on python 2. fix #67 --- pyexcel_io/service.py | 6 +++--- tests/test_service.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/pyexcel_io/service.py b/pyexcel_io/service.py index 2b7710c..d5ffc63 100644 --- a/pyexcel_io/service.py +++ b/pyexcel_io/service.py @@ -180,7 +180,7 @@ ODS_WRITE_FORMAT_COVERSION = { if PY2: ODS_WRITE_FORMAT_COVERSION[unicode] = "string" # noqa: F821 - ODS_WRITE_FORMAT_COVERSION[long] = "throw_exception" # noqa: F821 + ODS_WRITE_FORMAT_COVERSION[long] = "long" # noqa: F821 VALUE_CONVERTERS = { @@ -198,7 +198,7 @@ def throw_exception(value): def ods_float_value(value): - if int(value) > int(constants.MAX_INTEGER): + if value > constants.MAX_INTEGER: raise exceptions.IntegerAccuracyLossError("%s is too big" % value) return value @@ -234,7 +234,7 @@ ODS_VALUE_CONVERTERS = { "boolean": ods_bool_value, "timedelta": ods_timedelta_value, "float": ods_float_value, - "throw_exception": throw_exception + "long": ods_float_value } diff --git a/tests/test_service.py b/tests/test_service.py index 10b486e..60e5f1b 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,3 +1,4 @@ +import sys from nose.tools import eq_, raises from pyexcel_io.service import date_value, time_value from pyexcel_io.service import detect_int_value @@ -9,6 +10,8 @@ from pyexcel_io._compact import PY2 from pyexcel_io.exceptions import IntegerAccuracyLossError from nose import SkipTest +PY2 = sys.version[0] == 2 + def test_date_util_parse(): value = "2015-08-17T19:20:00" @@ -110,6 +113,21 @@ def test_big_int_value(): ods_float_value(1000000000000000) +def test_max_value_on_python_2(): + if PY2: + ods_float_value(long(999999999999999)) + else: + raise SkipTest("No long in python 3") + + +@raises(IntegerAccuracyLossError) +def test_really_long_value_on_python2(): + if PY2: + ods_float_value(long(999999999999999+1)) + else: + raise SkipTest("No long in python 3") + + @raises(IntegerAccuracyLossError) def test_throw_exception(): throw_exception(1000000000000000) From 5845728918548b3034d83c413af6f23c6564fa7b Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 19 Mar 2019 07:44:26 +0000 Subject: [PATCH 015/143] :egg: :ferris_wheel: release 0.5.16 --- .travis.yml | 20 ++++++++++++++++---- CHANGELOG.rst | 9 +++++++++ changelog.yml | 6 ++++++ docs/source/conf.py | 4 ++-- pyexcel-io.yml | 6 +++--- setup.py | 8 ++++---- 6 files changed, 40 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index c2c6b83..7f37b88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,18 +15,30 @@ python: stages: - test - lint + +.disable_global: &disable_global + before_install: false + install: true + before_script: false + after_success: false + after_failure: false + +.lint: &lint + <<: *disable_global + python: 3.6 + stage: lint + install: pip install flake8 + script: flake8 + jobs: include: - - stage: lint - python: 3.6 - script: make lint + - *lint stage: test script: make test before_install: - - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then mv min_requirements.txt requirements.txt ; fi diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 56917c6..961c5af 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.5.16 - 19.03.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#67 `_: fix conversion + issue for long type on python 2.7 for ods + 0.5.15 - 16.03.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index ed0bdce..2d0beae 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`#67`: fix conversion issue for long type on python 2.7 for ods' + version: 0.5.16 + date: 19.03.2019 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index bed6d04..5db4019 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,9 +26,9 @@ project = 'pyexcel-io' copyright = 'copyright 2015-2019 Onni Software Ltd.' author = 'Onni Software Ltd.' # The short X.Y version -version = '0.5.15' +version = '0.5.16' # The full version, including alpha/beta/rc tags -release = '0.5.15' +release = '0.5.16' # -- General configuration --------------------------------------------------- diff --git a/pyexcel-io.yml b/pyexcel-io.yml index f906d2f..06d261a 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,10 +2,10 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.5.15 -current_version: 0.5.15 +version: 0.5.16 +current_version: 0.5.16 copyright_year: 2015-2019 -release: 0.5.15 +release: 0.5.16 dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 diff --git a/setup.py b/setup.py index cf93e8d..a52976b 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = 'pyexcel-io' AUTHOR = 'C.W.' -VERSION = '0.5.15' +VERSION = '0.5.16' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'New BSD' DESCRIPTION = ( @@ -37,7 +37,7 @@ DESCRIPTION = ( 'format and to/from databases' ) URL = 'https://github.com/pyexcel/pyexcel-io' -DOWNLOAD_URL = '%s/archive/0.5.15.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.5.16.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -81,8 +81,8 @@ EXTRAS_REQUIRE = { # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs pyexcel-io v0.5.15 ' + - "Find 0.5.15 in changelog for more details") +GS_COMMAND = ('gs pyexcel-io v0.5.16 ' + + "Find 0.5.16 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( From c7a95d00410d559af6fe0f292cd6057d4c4d48de Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 19 Mar 2019 07:45:27 +0000 Subject: [PATCH 016/143] :green_heart: fix linting issue --- tests/test_service.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_service.py b/tests/test_service.py index 60e5f1b..1ea7466 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,4 +1,3 @@ -import sys from nose.tools import eq_, raises from pyexcel_io.service import date_value, time_value from pyexcel_io.service import detect_int_value @@ -10,8 +9,6 @@ from pyexcel_io._compact import PY2 from pyexcel_io.exceptions import IntegerAccuracyLossError from nose import SkipTest -PY2 = sys.version[0] == 2 - def test_date_util_parse(): value = "2015-08-17T19:20:00" From 1dfe9457f19aeb4b1e1f3548be00e99526dd4ca2 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 19 Mar 2019 07:48:40 +0000 Subject: [PATCH 017/143] :green_heart: update unit test --- tests/test_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_service.py b/tests/test_service.py index 1ea7466..1c234f1 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -100,7 +100,7 @@ def test_detect_float_value_on_custom_nan_text2(): def test_ods_write_format_conversion(): if PY2: expected = ODS_WRITE_FORMAT_COVERSION[long] # noqa: F821 - eq_('throw_exception', expected) + eq_('long', expected) else: raise SkipTest() From 935c8a05b62c76adc731ff746120d3c684a4c1a7 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 19 Mar 2019 07:57:07 +0000 Subject: [PATCH 018/143] :green_heart: fix travis command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7f37b88..43b2d43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ stages: python: 3.6 stage: lint install: pip install flake8 - script: flake8 + script: make lint jobs: include: From 086aacff85f9f09d4ba2925fd6d53b67c2061685 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 4 Apr 2019 21:26:36 +0100 Subject: [PATCH 019/143] :wheel_chair: better error message. fix #68 --- .travis.yml | 1 + CHANGELOG.rst | 9 +++++++++ changelog.yml | 6 ++++++ docs/source/conf.py | 4 ++-- pyexcel-io.yml | 6 +++--- pyexcel_io/io.py | 12 +++++++++++- setup.py | 10 +++++----- tests/test_io.py | 12 ++++++------ 8 files changed, 43 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 43b2d43..2787c2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ + sudo: false dist: xenial language: python diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 961c5af..8cb0e07 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.5.17 - 04.04.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#68 `_: Raise IOError when + the data file does not exist + 0.5.16 - 19.03.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index 2d0beae..472012f 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`#68`: Raise IOError when the data file does not exist' + version: 0.5.17 + date: 04.04.2019 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index 5db4019..5a4812c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,9 +26,9 @@ project = 'pyexcel-io' copyright = 'copyright 2015-2019 Onni Software Ltd.' author = 'Onni Software Ltd.' # The short X.Y version -version = '0.5.16' +version = '0.5.17' # The full version, including alpha/beta/rc tags -release = '0.5.16' +release = '0.5.17' # -- General configuration --------------------------------------------------- diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 06d261a..151b9cf 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,10 +2,10 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.5.16 -current_version: 0.5.16 +version: 0.5.17 +current_version: 0.5.17 copyright_year: 2015-2019 -release: 0.5.16 +release: 0.5.17 dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 4dd0e76..90af4ce 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -7,11 +7,13 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +import os from types import GeneratorType import warnings from pyexcel_io._compact import isstream, PY2 from pyexcel_io.plugins import READERS, WRITERS +from pyexcel_io.exceptions import NoSupportingPluginFound import pyexcel_io.constants as constants @@ -182,7 +184,15 @@ def load_data( except AttributeError: raise Exception("file_name should be a string type") - reader = READERS.get_a_plugin(file_type, library) + try: + reader = READERS.get_a_plugin(file_type, library) + except NoSupportingPluginFound: + if file_name: + if not os.path.exists(file_name): + raise IOError("%s does not exist" % file_name) + else: + raise + if file_name: reader.open(file_name, **keywords) elif file_content: diff --git a/setup.py b/setup.py index a52976b..9ce9243 100644 --- a/setup.py +++ b/setup.py @@ -29,15 +29,15 @@ except (ValueError, UnicodeError, locale.Error): NAME = 'pyexcel-io' AUTHOR = 'C.W.' -VERSION = '0.5.16' -EMAIL = 'wangc_2011@hotmail.com' +VERSION = '0.5.17' +EMAIL = 'info@pyexcel.org' LICENSE = 'New BSD' DESCRIPTION = ( 'A python library to read and write structured data in csv, zipped csv' + 'format and to/from databases' ) URL = 'https://github.com/pyexcel/pyexcel-io' -DOWNLOAD_URL = '%s/archive/0.5.16.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.5.17.tar.gz' % URL FILES = ['README.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -81,8 +81,8 @@ EXTRAS_REQUIRE = { # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs pyexcel-io v0.5.16 ' + - "Find 0.5.16 in changelog for more details") +GS_COMMAND = ('gs pyexcel-io v0.5.17 ' + + "Find 0.5.17 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( diff --git a/tests/test_io.py b/tests/test_io.py index 8670c18..39748cf 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -24,6 +24,11 @@ def test_force_file_type(): eq_(expected, data[test_file]) +@raises(IOError) +def test_invalid_file(): + load_data('/something/does/not/exist') + + @raises(IOError) def test_no_valid_parameters(): load_data() @@ -84,7 +89,7 @@ def test_write_xlsx_data_to_memory(): eq_(str(e), msg) -@raises(exceptions.NoSupportingPluginFound) +@raises(IOError) def test_load_unknown_data(): get_data("test.unknown") @@ -109,11 +114,6 @@ def test_write_xlsx_data(): get_data("test.xlsx") -@raises(exceptions.NoSupportingPluginFound) -def test_write_unknown_data(): - get_data("test.unknown") - - @raises(Exception) def test_writer_csvz_data_from_memory(): if not PY2: From ec62f384ef814d51d95bf2fb15b8a9b239249691 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 4 Apr 2019 21:42:02 +0100 Subject: [PATCH 020/143] :hammer: reformat code --- Makefile | 5 ++--- docs/source/conf.py | 2 +- pyexcel_io/__init__.py | 13 ++++++------ pyexcel_io/book.py | 1 + pyexcel_io/database/__init__.py | 3 +-- pyexcel_io/exceptions.py | 8 ++++--- pyexcel_io/io.py | 5 +++-- pyexcel_io/plugins.py | 8 +++---- pyexcel_io/readers/__init__.py | 1 - pyexcel_io/readers/csvz.py | 1 + pyexcel_io/readers/tsv.py | 1 + pyexcel_io/readers/tsvz.py | 1 + pyexcel_io/service.py | 5 ++--- pyexcel_io/writers/__init__.py | 1 - pyexcel_io/writers/csvz.py | 1 + pyexcel_io/writers/tsv.py | 1 + pyexcel_io/writers/tsvz.py | 1 + tests/test_base.py | 3 ++- tests/test_book.py | 3 ++- tests/test_csv_book.py | 3 ++- tests/test_django_book.py | 3 ++- tests/test_filter.py | 3 ++- tests/test_io.py | 5 +++-- tests/test_issues.py | 4 ++-- tests/test_new_csv_book.py | 3 ++- tests/test_new_csvz_book.py | 3 ++- tests/test_pyexcel_integration.py | 1 - tests/test_renderer.py | 3 ++- tests/test_service.py | 22 +++++++++++-------- tests/test_sheet.py | 3 ++- tests/test_sql_book.py | 35 ++++++++++++++++--------------- 31 files changed, 84 insertions(+), 68 deletions(-) diff --git a/Makefile b/Makefile index b857377..48ac0a6 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,12 @@ document: sphinx-autogen -o docs/source/generated/ docs/source/*.rst sphinx-build -b html docs/source/ docs/build/ -<<<<<<< HEAD format: isort -y $(find pyexcel_io -name "*.py"|xargs echo) $(find tests -name "*.py"|xargs echo) black -l 79 pyexcel_io black -l 79 tests -======= + lint: bash lint.sh ->>>>>>> master + diff --git a/docs/source/conf.py b/docs/source/conf.py index 21c4baf..37c4e22 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,4 +69,4 @@ html_static_path = ['static'] # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} \ No newline at end of file +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/pyexcel_io/__init__.py b/pyexcel_io/__init__.py index b50c3e3..3362892 100644 --- a/pyexcel_io/__init__.py +++ b/pyexcel_io/__init__.py @@ -8,13 +8,14 @@ :license: New BSD License, see LICENSE for more details """ import logging + +import pyexcel_io.plugins as plugins + +from .io import get_data, iget_data, save_data # noqa from ._compact import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) # noqa -from .io import get_data, iget_data, save_data # noqa -import pyexcel_io.plugins as plugins - BLACK_LIST = [__name__, "pyexcel_webio", "pyexcel_text"] WHITE_LIST = [ @@ -25,7 +26,5 @@ WHITE_LIST = [ PREFIX_PATTERN = "^pyexcel_.*$" plugins.load_plugins( - PREFIX_PATTERN, - __path__, # noqa: F821 - BLACK_LIST, - WHITE_LIST) + PREFIX_PATTERN, __path__, BLACK_LIST, WHITE_LIST # noqa: F821 +) diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index 21db709..5eb3072 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -9,6 +9,7 @@ """ import pyexcel_io.manager as manager from pyexcel_io._compact import PY2, OrderedDict, isstream + from .constants import MESSAGE_ERROR_03, MESSAGE_WRONG_IO_INSTANCE diff --git a/pyexcel_io/database/__init__.py b/pyexcel_io/database/__init__.py index e015023..addf50d 100644 --- a/pyexcel_io/database/__init__.py +++ b/pyexcel_io/database/__init__.py @@ -8,8 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import IOPluginInfoChain -from pyexcel_io.constants import DB_DJANGO, DB_SQL - +from pyexcel_io.constants import DB_SQL, DB_DJANGO IOPluginInfoChain(__name__).add_a_reader( relative_plugin_class_path="exporters.django.DjangoBookReader", diff --git a/pyexcel_io/exceptions.py b/pyexcel_io/exceptions.py index a971cb0..458ad2c 100644 --- a/pyexcel_io/exceptions.py +++ b/pyexcel_io/exceptions.py @@ -46,11 +46,13 @@ class IntegerAccuracyLossError(Exception): b=get_sheet(file_name='abc.ods') b[0,0] != s[0,0] """ + def __init__(self, message): custom_message = ( - message + '\n' + - "In order to keep its accuracy, please save as string. Then " + - "convert to int, long or float after the value will be read back" + message + + "\n" + + "In order to keep its accuracy, please save as string. Then " + + "convert to int, long or float after the value will be read back" ) super(IntegerAccuracyLossError, self).__init__(custom_message) diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index c02ab77..73646d1 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -11,8 +11,9 @@ import os import warnings from types import GeneratorType -from pyexcel_io._compact import isstream, PY2 +from pyexcel_io import constants from pyexcel_io.plugins import READERS, WRITERS +from pyexcel_io._compact import PY2, isstream from pyexcel_io.exceptions import NoSupportingPluginFound @@ -124,7 +125,7 @@ def save_data(afile, data, file_type=None, **keywords): keywords.update(dict(file_stream=afile, file_type=file_type)) else: keywords.update(dict(file_name=afile, file_type=file_type)) - keywords['single_sheet_in_book'] = single_sheet_in_book + keywords["single_sheet_in_book"] = single_sheet_in_book with get_writer(**keywords) as writer: writer.write(to_store) diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index b36047c..2c350ef 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -7,14 +7,12 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from lml.loader import scan_plugins_regex -from lml.plugin import PluginManager -from lml.plugin import PluginInfoChain, PluginInfo - import pyexcel_io.utils as ioutils import pyexcel_io.manager as manager import pyexcel_io.constants as constants import pyexcel_io.exceptions as exceptions +from lml.loader import scan_plugins_regex +from lml.plugin import PluginInfo, PluginManager, PluginInfoChain ERROR_MESSAGE_FORMATTER = "one of these plugins for %s data in '%s': %s" UPGRADE_MESSAGE = "Please upgrade the plugin '%s' according to \ @@ -137,5 +135,5 @@ def load_plugins(plugin_name_patterns, path, black_list, white_list): plugin_name_patterns=plugin_name_patterns, pyinstaller_path=path, black_list=black_list, - white_list=white_list + white_list=white_list, ) diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index 59bcfea..754e022 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -9,7 +9,6 @@ """ from pyexcel_io.plugins import IOPluginInfoChain - IOPluginInfoChain(__name__).add_a_reader( relative_plugin_class_path="csvr.CSVBookReader", file_types=["csv"], diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index 24410b1..c956eea 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -12,6 +12,7 @@ import zipfile from pyexcel_io.book import BookReader from pyexcel_io._compact import PY2, StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ + from .csvr import NamedContent, CSVinMemoryReader diff --git a/pyexcel_io/readers/tsv.py b/pyexcel_io/readers/tsv.py index 8ac135a..35829ae 100644 --- a/pyexcel_io/readers/tsv.py +++ b/pyexcel_io/readers/tsv.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants + from .csvr import CSVBookReader diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py index c7b7310..9dc7d6d 100644 --- a/pyexcel_io/readers/tsvz.py +++ b/pyexcel_io/readers/tsvz.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT + from .csvz import CSVZipBookReader diff --git a/pyexcel_io/service.py b/pyexcel_io/service.py index d5ffc63..a074c27 100644 --- a/pyexcel_io/service.py +++ b/pyexcel_io/service.py @@ -11,9 +11,8 @@ import re import math import datetime +from pyexcel_io import constants, exceptions from pyexcel_io._compact import PY2 -from pyexcel_io import constants -from pyexcel_io import exceptions def has_no_digits_in_float(value): @@ -234,7 +233,7 @@ ODS_VALUE_CONVERTERS = { "boolean": ods_bool_value, "timedelta": ods_timedelta_value, "float": ods_float_value, - "long": ods_float_value + "long": ods_float_value, } diff --git a/pyexcel_io/writers/__init__.py b/pyexcel_io/writers/__init__.py index bc5d767..30f62eb 100644 --- a/pyexcel_io/writers/__init__.py +++ b/pyexcel_io/writers/__init__.py @@ -9,7 +9,6 @@ """ from pyexcel_io.plugins import IOPluginInfoChain - IOPluginInfoChain(__name__).add_a_writer( relative_plugin_class_path="csvw.CSVBookWriter", file_types=["csv"], diff --git a/pyexcel_io/writers/csvz.py b/pyexcel_io/writers/csvz.py index 76d291b..3c3c3de 100644 --- a/pyexcel_io/writers/csvz.py +++ b/pyexcel_io/writers/csvz.py @@ -12,6 +12,7 @@ import zipfile from pyexcel_io.book import BookWriter from pyexcel_io._compact import PY2, StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ, DEFAULT_SHEET_NAME + from .csvw import UnicodeWriter, CSVSheetWriter diff --git a/pyexcel_io/writers/tsv.py b/pyexcel_io/writers/tsv.py index 778372b..757cad2 100644 --- a/pyexcel_io/writers/tsv.py +++ b/pyexcel_io/writers/tsv.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants + from .csvw import CSVBookWriter diff --git a/pyexcel_io/writers/tsvz.py b/pyexcel_io/writers/tsvz.py index 255730a..b12814d 100644 --- a/pyexcel_io/writers/tsvz.py +++ b/pyexcel_io/writers/tsvz.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT + from .csvz import CSVZipBookWriter diff --git a/tests/test_base.py b/tests/test_base.py index a7c890e..df4dfb1 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,8 +1,9 @@ -from nose.tools import raises from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetReader, SheetWriter, NamedContent from pyexcel_io.utils import is_empty_array +from nose.tools import raises + @raises(NotImplementedError) def test_book_writer(): diff --git a/tests/test_book.py b/tests/test_book.py index 0e98e57..70ad3c3 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -1,6 +1,7 @@ -from nose.tools import raises from pyexcel_io.book import BookReader, BookWriter, RWInterface +from nose.tools import raises + @raises(NotImplementedError) def test_rwinterface(): diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index c530f7b..ecbacb6 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -6,7 +6,6 @@ from textwrap import dedent from unittest import TestCase import pyexcel_io.manager as manager -from nose.tools import eq_, raises from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import PY2, BytesIO, StringIO from pyexcel_io.readers.csvr import ( @@ -16,6 +15,8 @@ from pyexcel_io.readers.csvr import ( ) from pyexcel_io.writers.csvw import CSVFileWriter, CSVMemoryWriter +from nose.tools import eq_, raises + class TestReaders(TestCase): def setUp(self): diff --git a/tests/test_django_book.py b/tests/test_django_book.py index 95239df..7ad4a6a 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -1,4 +1,3 @@ -from nose.tools import eq_, raises from pyexcel_io import save_data from pyexcel_io._compact import OrderedDict from pyexcel_io.constants import DB_DJANGO @@ -17,6 +16,8 @@ from pyexcel_io.database.importers.django import ( DjangoModelWriter, ) +from nose.tools import eq_, raises + class Package: def __init__(self, raiseException=False, **keywords): diff --git a/tests/test_filter.py b/tests/test_filter.py index 3af8eac..a3d3591 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -1,10 +1,11 @@ import os import pyexcel_io.constants as constants -from nose.tools import eq_ from pyexcel_io import get_data, save_data from pyexcel_io.utils import _index_filter +from nose.tools import eq_ + def test_index_filter(): current_index, start, limit, expected = (0, 1, -1, constants.SKIP_DATA) diff --git a/tests/test_io.py b/tests/test_io.py index 298eb65..ae7d148 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -6,11 +6,12 @@ from unittest import TestCase import pyexcel_io.manager as manager import pyexcel_io.exceptions as exceptions -from nose.tools import eq_, raises from pyexcel_io import get_data, iget_data, save_data from pyexcel_io.io import load_data, get_writer from pyexcel_io._compact import BytesIO, StringIO, OrderedDict, is_string +from nose.tools import eq_, raises + PY2 = sys.version_info[0] == 2 @@ -25,7 +26,7 @@ def test_force_file_type(): @raises(IOError) def test_invalid_file(): - load_data('/something/does/not/exist') + load_data("/something/does/not/exist") @raises(IOError) diff --git a/tests/test_issues.py b/tests/test_issues.py index 7a436d2..09582e0 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -4,11 +4,11 @@ import os import pyexcel as p +from pyexcel_io import get_data, save_data +from pyexcel_io._compact import PY26 from nose import SkipTest from nose.tools import eq_ -from pyexcel_io import get_data, save_data -from pyexcel_io._compact import PY26 IN_TRAVIS = "TRAVIS" in os.environ diff --git a/tests/test_new_csv_book.py b/tests/test_new_csv_book.py index 857f4ee..930a58a 100644 --- a/tests/test_new_csv_book.py +++ b/tests/test_new_csv_book.py @@ -3,13 +3,14 @@ from textwrap import dedent from unittest import TestCase import pyexcel_io.manager as manager -from nose.tools import raises from pyexcel_io._compact import OrderedDict from pyexcel_io.readers.tsv import TSVBookReader from pyexcel_io.writers.tsv import TSVBookWriter from pyexcel_io.readers.csvr import CSVBookReader from pyexcel_io.writers.csvw import CSVBookWriter +from nose.tools import raises + class TestCSVReaders(TestCase): file_type = "csv" diff --git a/tests/test_new_csvz_book.py b/tests/test_new_csvz_book.py index fdf8620..b73faed 100644 --- a/tests/test_new_csvz_book.py +++ b/tests/test_new_csvz_book.py @@ -5,7 +5,6 @@ import zipfile from unittest import TestCase import pyexcel_io.manager as manager -from nose.tools import raises from pyexcel_io import save_data from pyexcel_io._compact import OrderedDict from pyexcel_io.readers.csvz import CSVZipBookReader @@ -13,6 +12,8 @@ from pyexcel_io.readers.tsvz import TSVZipBookReader from pyexcel_io.writers.csvz import CSVZipBookWriter from pyexcel_io.writers.tsvz import TSVZipBookWriter +from nose.tools import raises + PY2 = sys.version_info[0] == 2 diff --git a/tests/test_pyexcel_integration.py b/tests/test_pyexcel_integration.py index e198c74..963da21 100644 --- a/tests/test_pyexcel_integration.py +++ b/tests/test_pyexcel_integration.py @@ -4,7 +4,6 @@ from textwrap import dedent from unittest import TestCase import pyexcel as pe - from pyexcel_io._compact import text_type diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 5be2174..209bad9 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -1,8 +1,9 @@ import os -from nose.tools import eq_ from pyexcel_io import get_data, save_data +from nose.tools import eq_ + class TestRenderer: def setUp(self): diff --git a/tests/test_service.py b/tests/test_service.py index 1c234f1..5971708 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,13 +1,17 @@ -from nose.tools import eq_, raises -from pyexcel_io.service import date_value, time_value -from pyexcel_io.service import detect_int_value -from pyexcel_io.service import detect_float_value -from pyexcel_io.service import ODS_WRITE_FORMAT_COVERSION -from pyexcel_io.service import ods_float_value -from pyexcel_io.service import throw_exception +from pyexcel_io.service import ( + ODS_WRITE_FORMAT_COVERSION, + date_value, + time_value, + ods_float_value, + throw_exception, + detect_int_value, + detect_float_value, +) from pyexcel_io._compact import PY2 from pyexcel_io.exceptions import IntegerAccuracyLossError + from nose import SkipTest +from nose.tools import eq_, raises def test_date_util_parse(): @@ -100,7 +104,7 @@ def test_detect_float_value_on_custom_nan_text2(): def test_ods_write_format_conversion(): if PY2: expected = ODS_WRITE_FORMAT_COVERSION[long] # noqa: F821 - eq_('long', expected) + eq_("long", expected) else: raise SkipTest() @@ -120,7 +124,7 @@ def test_max_value_on_python_2(): @raises(IntegerAccuracyLossError) def test_really_long_value_on_python2(): if PY2: - ods_float_value(long(999999999999999+1)) + ods_float_value(long(999999999999999 + 1)) else: raise SkipTest("No long in python 3") diff --git a/tests/test_sheet.py b/tests/test_sheet.py index 789317a..d82cb30 100644 --- a/tests/test_sheet.py +++ b/tests/test_sheet.py @@ -1,7 +1,8 @@ import pyexcel_io.constants as constants -from nose.tools import eq_ from pyexcel_io.sheet import SheetReader, SheetWriter +from nose.tools import eq_ + class MyWriter(SheetWriter): def set_size(self, size): diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 6927056..47bc5c9 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -3,6 +3,24 @@ import json import datetime import platform +from pyexcel_io._compact import OrderedDict +from pyexcel_io.database.common import ( + SQLTableExporter, + SQLTableImporter, + SQLTableExportAdapter, + SQLTableImportAdapter, +) +from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.database.exporters.sqlalchemy import ( + SQLBookReader, + SQLTableReader, +) +from pyexcel_io.database.importers.sqlalchemy import ( + SQLBookWriter, + SQLTableWriter, + PyexcelSQLSkipRowException, +) + from nose.tools import eq_, raises from sqlalchemy import ( Date, @@ -15,24 +33,7 @@ from sqlalchemy import ( create_engine, ) from sqlalchemy.orm import backref, relationship, sessionmaker -from pyexcel_io._compact import OrderedDict -from pyexcel_io.database.common import ( - SQLTableExporter, - SQLTableImporter, - SQLTableExportAdapter, - SQLTableImportAdapter, -) from sqlalchemy.ext.declarative import declarative_base -from pyexcel_io.database.querysets import QuerysetsReader -from pyexcel_io.database.exporters.sqlalchemy import ( - SQLBookReader, - SQLTableReader, -) -from pyexcel_io.database.importers.sqlalchemy import ( - SQLBookWriter, - SQLTableWriter, - PyexcelSQLSkipRowException, -) PY3 = sys.version_info[0] == 3 PY36 = PY3 and sys.version_info[1] == 6 From b0ae275ecacddac57ca6ef98d95cf15cd9997b94 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 12 Jun 2019 20:51:48 +0100 Subject: [PATCH 021/143] :sparkles: support force_file_type on save_data. fix #69 --- CHANGELOG.rst | 9 ++++ changelog.yml | 6 +++ docs/source/conf.py | 8 +-- pyexcel-io.yml | 6 +-- pyexcel_io/io.py | 21 +++++--- setup.py | 120 +++++++++++++++++++++++--------------------- tests/test_io.py | 15 ++++++ 7 files changed, 113 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8cb0e07..7746689 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.5.18 - 12.06.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#69 `_: Force file + type(force_file_type) on write + 0.5.17 - 04.04.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index 472012f..e1d06e4 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`#69`: Force file type(force_file_type) on write' + version: 0.5.18 + date: 12.06.2019 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index 5a4812c..88e82dc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,12 +23,12 @@ DESCRIPTION = ( # -- Project information ----------------------------------------------------- project = 'pyexcel-io' -copyright = 'copyright 2015-2019 Onni Software Ltd.' -author = 'Onni Software Ltd.' +copyright = '2015-2019 Onni Software Ltd.' +author = 'C.W.' # The short X.Y version -version = '0.5.17' +version = '0.5.18' # The full version, including alpha/beta/rc tags -release = '0.5.17' +release = '0.5.18' # -- General configuration --------------------------------------------------- diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 151b9cf..ee1a6f0 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,10 +2,10 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.5.17 -current_version: 0.5.17 +version: 0.5.18 +current_version: 0.5.18 +release: 0.5.18 copyright_year: 2015-2019 -release: 0.5.17 dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 90af4ce..136212f 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -4,7 +4,7 @@ The io interface to file extensions - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2019 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import os @@ -24,7 +24,7 @@ def iget_data(afile, file_type=None, **keywords): :param sheet_name: the name of the sheet to be loaded :param sheet_index: the index of the sheet to be loaded :param sheets: a list of sheet to be loaded - :param file_type: used only when filename is not a physial file name + :param file_type: used only when filename is not a physical file name :param force_file_type: used only when filename refers to a physical file and it is intended to open it as forced file type. :param streaming: toggles the type of returned data. The values of the @@ -99,6 +99,8 @@ def save_data(afile, data, file_type=None, **keywords): :param filename: actual file name, a file stream or actual content :param data: a dictionary but an ordered dictionary is preferred :param file_type: used only when filename is not a physial file name + :param force_file_type: used only when filename refers to a physical file + and it is intended to open it as forced file type. :param library: explicitly name a library for use. e.g. library='pyexcel-ods' :param keywords: any other parameters that python csv module's @@ -217,7 +219,8 @@ def load_data( def get_writer( - file_name=None, file_stream=None, file_type=None, library=None, **keywords + file_name=None, file_stream=None, file_type=None, + library=None, force_file_type=None, **keywords ): """find a suitable writer""" inputs = [file_name, file_stream] @@ -227,11 +230,15 @@ def get_writer( raise IOError(constants.MESSAGE_ERROR_02) file_type_given = True + if file_type is None and file_name: - try: - file_type = file_name.split(".")[-1] - except AttributeError: - raise Exception("file_name should be a string type") + if force_file_type: + file_type = force_file_type + else: + try: + file_type = file_name.split(".")[-1] + except AttributeError: + raise Exception("file_name should be a string type") file_type_given = False diff --git a/setup.py b/setup.py index 9ce9243..6329fc5 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 # Template by pypi-mobans -import os -import sys import codecs import locale +import os import platform +import sys from shutil import rmtree -from setuptools import Command, setup, find_packages +from setuptools import Command, find_packages, setup PY2 = sys.version_info[0] == 2 PY26 = PY2 and sys.version_info[1] < 7 @@ -22,69 +22,73 @@ PY33 = sys.version_info < (3, 4) try: lc = locale.getlocale() pf = platform.system() - if pf != 'Windows' and lc == (None, None): - locale.setlocale(locale.LC_ALL, 'C.UTF-8') + if pf != "Windows" and lc == (None, None): + locale.setlocale(locale.LC_ALL, "C.UTF-8") except (ValueError, UnicodeError, locale.Error): - locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') + locale.setlocale(locale.LC_ALL, "en_US.UTF-8") -NAME = 'pyexcel-io' -AUTHOR = 'C.W.' -VERSION = '0.5.17' -EMAIL = 'info@pyexcel.org' -LICENSE = 'New BSD' +NAME = "pyexcel-io" +AUTHOR = "C.W." +VERSION = "0.5.18" +EMAIL = "info@pyexcel.org" +LICENSE = "New BSD" DESCRIPTION = ( - 'A python library to read and write structured data in csv, zipped csv' + - 'format and to/from databases' + "A python library to read and write structured data in csv, zipped csv" + + "format and to/from databases" ) -URL = 'https://github.com/pyexcel/pyexcel-io' -DOWNLOAD_URL = '%s/archive/0.5.17.tar.gz' % URL -FILES = ['README.rst', 'CHANGELOG.rst'] +URL = "https://github.com/pyexcel/pyexcel-io" +DOWNLOAD_URL = "%s/archive/0.5.18.tar.gz" % URL +FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ - 'python', - 'API', - 'tsv', - 'tsvz', - 'csv', - 'csvz', - 'django', - 'sqlalchemy', + "python", + "API", + "tsv", + "tsvz", + "csv", + "csvz", + "django", + "sqlalchemy", ] CLASSIFIERS = [ - 'Topic :: Software Development :: Libraries', - 'Programming Language :: Python', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', + "Topic :: Software Development :: Libraries", + "Programming Language :: Python", + "Intended Audience :: Developers", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + + "Programming Language :: Python :: 3.7", + + "Programming Language :: Python :: 3.8", + 'License :: OSI Approved :: BSD License', 'Programming Language :: Python :: Implementation :: PyPy' ] INSTALL_REQUIRES = [ - 'lml>=0.0.4', + "lml>=0.0.4", ] SETUP_COMMANDS = {} if PY26: INSTALL_REQUIRES.append('ordereddict') -PACKAGES = find_packages(exclude=['ez_setup', 'examples', 'tests']) +PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests"]) EXTRAS_REQUIRE = { - 'xls': ['pyexcel-xls>=0.5.0'], - 'xlsx': ['pyexcel-xlsx>=0.5.0'], - 'ods': ['pyexcel-ods3>=0.5.0'], + "xls": ['pyexcel-xls>=0.5.0'], + "xlsx": ['pyexcel-xlsx>=0.5.0'], + "ods": ['pyexcel-ods3>=0.5.0'], } # You do not need to read beyond this line -PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( - sys.executable) -GS_COMMAND = ('gs pyexcel-io v0.5.17 ' + - "Find 0.5.17 in changelog for more details") -NO_GS_MESSAGE = ('Automatic github release is disabled. ' + - 'Please install gease to enable it.') +PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) +GS_COMMAND = ("gs pyexcel-io v0.5.18 " + + "Find 0.5.18 in changelog for more details") +NO_GS_MESSAGE = ("Automatic github release is disabled. " + + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) HERE = os.path.abspath(os.path.dirname(__file__)) @@ -93,13 +97,13 @@ HERE = os.path.abspath(os.path.dirname(__file__)) class PublishCommand(Command): """Support setup.py upload.""" - description = 'Build and publish the package on github and pypi' + description = "Build and publish the package on github and pypi" user_options = [] @staticmethod def status(s): """Prints things in bold.""" - print('\033[1m{0}\033[0m'.format(s)) + print("\033[1m{0}\033[0m".format(s)) def initialize_options(self): pass @@ -109,14 +113,14 @@ class PublishCommand(Command): def run(self): try: - self.status('Removing previous builds...') - rmtree(os.path.join(HERE, 'dist')) - rmtree(os.path.join(HERE, 'build')) - rmtree(os.path.join(HERE, 'pyexcel_io.egg-info')) + self.status("Removing previous builds...") + rmtree(os.path.join(HERE, "dist")) + rmtree(os.path.join(HERE, "build")) + rmtree(os.path.join(HERE, "pyexcel_io.egg-info")) except OSError: pass - self.status('Building Source and Wheel (universal) distribution...') + self.status("Building Source and Wheel (universal) distribution...") run_status = True if has_gease(): run_status = os.system(GS_COMMAND) == 0 @@ -130,7 +134,7 @@ class PublishCommand(Command): SETUP_COMMANDS.update({ - 'publish': PublishCommand + "publish": PublishCommand }) @@ -159,7 +163,7 @@ def read_files(*files): def read(afile): """Read a file into setup""" the_relative_file = os.path.join(HERE, afile) - with codecs.open(the_relative_file, 'r', 'utf-8') as opened_file: + with codecs.open(the_relative_file, "r", "utf-8") as opened_file: content = filter_out_test_code(opened_file) content = "".join(list(content)) return content @@ -168,11 +172,11 @@ def read(afile): def filter_out_test_code(file_handle): found_test_code = False for line in file_handle.readlines(): - if line.startswith('.. testcode:'): + if line.startswith(".. testcode:"): found_test_code = True continue if found_test_code is True: - if line.startswith(' '): + if line.startswith(" "): continue else: empty_line = line.strip() @@ -182,14 +186,14 @@ def filter_out_test_code(file_handle): found_test_code = False yield line else: - for keyword in ['|version|', '|today|']: + for keyword in ["|version|", "|today|"]: if keyword in line: break else: yield line -if __name__ == '__main__': +if __name__ == "__main__": setup( test_suite="tests", name=NAME, @@ -203,7 +207,7 @@ if __name__ == '__main__': license=LICENSE, keywords=KEYWORDS, extras_require=EXTRAS_REQUIRE, - tests_require=['nose'], + tests_require=["nose"], install_requires=INSTALL_REQUIRES, packages=PACKAGES, include_package_data=True, diff --git a/tests/test_io.py b/tests/test_io.py index 39748cf..cca6af0 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -24,6 +24,21 @@ def test_force_file_type(): eq_(expected, data[test_file]) +def test_force_file_type_on_write(): + test_file = "force_file_type_on_write.txt" + save_data( + test_file, + {"sheet 1": [[1, 2]]}, + force_file_type="csv" + ) + data = get_data( + test_file, force_file_type="csv" + ) + expected = [[1, 2]] + eq_(expected, data[test_file]) + os.unlink(test_file) + + @raises(IOError) def test_invalid_file(): load_data('/something/does/not/exist') From 6f2660f0323767df640513d936d8aaa56437f70e Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 12 Jun 2019 20:59:18 +0100 Subject: [PATCH 022/143] :green_heart: make lint happy --- setup.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index 6329fc5..8615fd4 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,8 @@ VERSION = "0.5.18" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( - "A python library to read and write structured data in csv, zipped csv" + - "format and to/from databases" + "A python library to read and write structured data in csv, zipped csv" + + "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" DOWNLOAD_URL = "%s/archive/0.5.18.tar.gz" % URL @@ -60,37 +60,38 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: Implementation :: PyPy' + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: Implementation :: PyPy", ] -INSTALL_REQUIRES = [ - "lml>=0.0.4", -] +INSTALL_REQUIRES = ["lml>=0.0.4"] SETUP_COMMANDS = {} if PY26: - INSTALL_REQUIRES.append('ordereddict') + INSTALL_REQUIRES.append("ordereddict") PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests"]) EXTRAS_REQUIRE = { - "xls": ['pyexcel-xls>=0.5.0'], - "xlsx": ['pyexcel-xlsx>=0.5.0'], - "ods": ['pyexcel-ods3>=0.5.0'], + "xls": ["pyexcel-xls>=0.5.0"], + "xlsx": ["pyexcel-xlsx>=0.5.0"], + "ods": ["pyexcel-ods3>=0.5.0"], } # You do not need to read beyond this line -PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs pyexcel-io v0.5.18 " + - "Find 0.5.18 in changelog for more details") -NO_GS_MESSAGE = ("Automatic github release is disabled. " + - "Please install gease to enable it.") +PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format( + sys.executable +) +GS_COMMAND = ( + "gs pyexcel-io v0.5.18 " + "Find 0.5.18 in changelog for more details" +) +NO_GS_MESSAGE = ( + "Automatic github release is disabled. " + + "Please install gease to enable it." +) UPLOAD_FAILED_MSG = ( - 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) + 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND +) HERE = os.path.abspath(os.path.dirname(__file__)) @@ -133,9 +134,7 @@ class PublishCommand(Command): sys.exit() -SETUP_COMMANDS.update({ - "publish": PublishCommand -}) +SETUP_COMMANDS.update({"publish": PublishCommand}) def has_gease(): @@ -146,6 +145,7 @@ def has_gease(): """ try: import gease # noqa + return True except ImportError: return False @@ -213,5 +213,5 @@ if __name__ == "__main__": include_package_data=True, zip_safe=False, classifiers=CLASSIFIERS, - cmdclass=SETUP_COMMANDS + cmdclass=SETUP_COMMANDS, ) From 57b1b49b0a177a75c43443dde19819295125f97e Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 14 Jul 2019 08:08:56 +0100 Subject: [PATCH 023/143] :bug: fix python 3 bug with http source. fix https://github.com/pyexcel/pyexcel/issues/185 --- pyexcel_io/book.py | 6 ++++++ tests/test_book.py | 22 +++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index 24e9b96..ab26744 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -231,6 +231,12 @@ class BookWriter(RWInterface): def _convert_content_to_stream(file_content, file_type): stream = manager.get_io(file_type) + if not PY2: + target_content_type = manager.get_io_type(file_type) + if target_content_type == 'bytes' and not isinstance(file_content, bytes): + file_content = file_content.encode('utf-8') + elif target_content_type == 'string' and isinstance(file_content, bytes): + file_content = file_content.decode('utf-8') stream.write(file_content) stream.seek(0) return stream diff --git a/tests/test_book.py b/tests/test_book.py index 226c6ca..4256490 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -1,5 +1,7 @@ from nose.tools import raises -from pyexcel_io.book import RWInterface, BookReader, BookWriter +from pyexcel_io.book import RWInterface, BookReader, BookWriter, _convert_content_to_stream +from pyexcel_io._compact import PY2, StringIO, BytesIO +from nose import SkipTest @raises(NotImplementedError) @@ -30,3 +32,21 @@ def test_book_reader_open_stream(): def test_book_writer(): writer = BookWriter() writer.open_stream("a string") + + +def test_convert_to_bytes_stream(): + if PY2: + raise SkipTest('No need test in python 2') + else: + file_content = b'test' + stream = _convert_content_to_stream(file_content, 'string') + assert isinstance(stream, StringIO) + + +def test_convert_to_string_stream(): + if PY2: + raise SkipTest('No need test in python 2') + else: + file_content = 'test' + stream = _convert_content_to_stream(file_content, 'bytes') + assert isinstance(stream, BytesIO) From 3a6e86cfd6ba5a5805a5ce8fa84761a9d4e63f68 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 14 Jul 2019 08:13:55 +0100 Subject: [PATCH 024/143] :green_heart: text and binary file types --- tests/test_book.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_book.py b/tests/test_book.py index 4256490..d196887 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -39,7 +39,7 @@ def test_convert_to_bytes_stream(): raise SkipTest('No need test in python 2') else: file_content = b'test' - stream = _convert_content_to_stream(file_content, 'string') + stream = _convert_content_to_stream(file_content, 'csv') assert isinstance(stream, StringIO) @@ -48,5 +48,5 @@ def test_convert_to_string_stream(): raise SkipTest('No need test in python 2') else: file_content = 'test' - stream = _convert_content_to_stream(file_content, 'bytes') + stream = _convert_content_to_stream(file_content, 'csvz') assert isinstance(stream, BytesIO) From 84ec6199fbdbb0555ca13106d2a79733c1e88ae2 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 14 Jul 2019 08:17:35 +0100 Subject: [PATCH 025/143] :books: update docs and prepare for an incremental release --- .travis.yml | 29 +++++++++++++++++++------ CHANGELOG.rst | 9 ++++++++ changelog.yml | 6 ++++++ docs/source/conf.py | 4 ++-- pyexcel-io.yml | 6 +++--- setup.py | 52 ++++++++++++++++++++++----------------------- 6 files changed, 69 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2787c2e..51c299b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ - sudo: false dist: xenial language: python @@ -16,34 +15,52 @@ python: stages: - test - lint + - moban .disable_global: &disable_global + addons: false + cache: false + env: {} + python: false before_install: false - install: true + install: false before_script: false + script: false after_success: false after_failure: false + before_deploy: false + deploy: false .lint: &lint <<: *disable_global + git: + submodules: false python: 3.6 stage: lint - install: pip install flake8 script: make lint +.moban: &moban + <<: *disable_global + python: 3.6 + stage: moban + install: pip install moban>=0.0.4 + script: + - moban + - git diff --exit-code + jobs: include: + - *moban - *lint stage: test -script: make test - before_install: - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then mv min_requirements.txt requirements.txt ; fi - - test ! -f rnd_requirements.txt || pip install --no-deps -r rnd_requirements.txt + - test ! -f rnd_requirements.txt || + pip install --no-deps -r rnd_requirements.txt - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; - pip install -r tests/requirements.txt script: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7746689..60f7687 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.5.19 - 14.7.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `pyexcel#185 `_: handle stream + conversion if file type(html) needs string content then bytes to handle + 0.5.18 - 12.06.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index e1d06e4..d26e841 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`pyexcel#185`: handle stream conversion if file type(html) needs string content then bytes to handle' + version: 0.5.19 + date: 14.7.2019 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index 88e82dc..bd43399 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,9 +26,9 @@ project = 'pyexcel-io' copyright = '2015-2019 Onni Software Ltd.' author = 'C.W.' # The short X.Y version -version = '0.5.18' +version = '0.5.19' # The full version, including alpha/beta/rc tags -release = '0.5.18' +release = '0.5.19' # -- General configuration --------------------------------------------------- diff --git a/pyexcel-io.yml b/pyexcel-io.yml index ee1a6f0..2597ec3 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,9 +2,9 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.5.18 -current_version: 0.5.18 -release: 0.5.18 +version: 0.5.19 +current_version: 0.5.19 +release: 0.5.19 copyright_year: 2015-2019 dependencies: - ordereddict;python_version<"2.7" diff --git a/setup.py b/setup.py index 8615fd4..bb0987f 100644 --- a/setup.py +++ b/setup.py @@ -29,15 +29,15 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "C.W." -VERSION = "0.5.18" +VERSION = "0.5.19" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( - "A python library to read and write structured data in csv, zipped csv" - + "format and to/from databases" + "A python library to read and write structured data in csv, zipped csv" + + "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.5.18.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.5.19.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -60,38 +60,37 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: Implementation :: PyPy", + + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python :: Implementation :: PyPy' ] -INSTALL_REQUIRES = ["lml>=0.0.4"] +INSTALL_REQUIRES = [ + "lml>=0.0.4", +] SETUP_COMMANDS = {} if PY26: - INSTALL_REQUIRES.append("ordereddict") + INSTALL_REQUIRES.append('ordereddict') PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests"]) EXTRAS_REQUIRE = { - "xls": ["pyexcel-xls>=0.5.0"], - "xlsx": ["pyexcel-xlsx>=0.5.0"], - "ods": ["pyexcel-ods3>=0.5.0"], + "xls": ['pyexcel-xls>=0.5.0'], + "xlsx": ['pyexcel-xlsx>=0.5.0'], + "ods": ['pyexcel-ods3>=0.5.0'], } # You do not need to read beyond this line -PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format( - sys.executable -) -GS_COMMAND = ( - "gs pyexcel-io v0.5.18 " + "Find 0.5.18 in changelog for more details" -) -NO_GS_MESSAGE = ( - "Automatic github release is disabled. " - + "Please install gease to enable it." -) +PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) +GS_COMMAND = ("gs pyexcel-io v0.5.19 " + + "Find 0.5.19 in changelog for more details") +NO_GS_MESSAGE = ("Automatic github release is disabled. " + + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( - 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND -) + 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) HERE = os.path.abspath(os.path.dirname(__file__)) @@ -134,7 +133,9 @@ class PublishCommand(Command): sys.exit() -SETUP_COMMANDS.update({"publish": PublishCommand}) +SETUP_COMMANDS.update({ + "publish": PublishCommand +}) def has_gease(): @@ -145,7 +146,6 @@ def has_gease(): """ try: import gease # noqa - return True except ImportError: return False @@ -213,5 +213,5 @@ if __name__ == "__main__": include_package_data=True, zip_safe=False, classifiers=CLASSIFIERS, - cmdclass=SETUP_COMMANDS, + cmdclass=SETUP_COMMANDS ) From 3b6a380b70b53f6fb86d5c24f2db9d9c9cbe92e9 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 14 Jul 2019 08:21:12 +0100 Subject: [PATCH 026/143] :green_heart: fix styling problem --- pyexcel_io/book.py | 8 ++++++-- tests/test_book.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index ab26744..dbcbc32 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -233,9 +233,13 @@ def _convert_content_to_stream(file_content, file_type): stream = manager.get_io(file_type) if not PY2: target_content_type = manager.get_io_type(file_type) - if target_content_type == 'bytes' and not isinstance(file_content, bytes): + needs_encode = (target_content_type == 'bytes' and + not isinstance(file_content, bytes)) + needs_decode = (target_content_type == 'string' and + isinstance(file_content, bytes)) + if needs_encode: file_content = file_content.encode('utf-8') - elif target_content_type == 'string' and isinstance(file_content, bytes): + elif needs_decode: file_content = file_content.decode('utf-8') stream.write(file_content) stream.seek(0) diff --git a/tests/test_book.py b/tests/test_book.py index d196887..6d9f538 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -1,5 +1,6 @@ from nose.tools import raises -from pyexcel_io.book import RWInterface, BookReader, BookWriter, _convert_content_to_stream +from pyexcel_io.book import RWInterface, BookReader, BookWriter +from pyexcel_io.book import _convert_content_to_stream from pyexcel_io._compact import PY2, StringIO, BytesIO from nose import SkipTest From c964d18b27d29fe7b7834c2e995c8bed66280b3e Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 14 Jul 2019 08:22:38 +0100 Subject: [PATCH 027/143] :fire: remove unused moban section --- .travis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 51c299b..2b2d447 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,15 +39,6 @@ stages: stage: lint script: make lint -.moban: &moban - <<: *disable_global - python: 3.6 - stage: moban - install: pip install moban>=0.0.4 - script: - - moban - - git diff --exit-code - jobs: include: - *moban From 1478f083e7de9eaeaa01aa3c9f50bdee4fee9df2 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 14 Jul 2019 08:24:50 +0100 Subject: [PATCH 028/143] :fire: remove moban totally --- .travis.yml | 1 - pyexcel-io.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2b2d447..734bd84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ python: stages: - test - lint - - moban .disable_global: &disable_global addons: false diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 2597ec3..9cd365c 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -6,6 +6,7 @@ version: 0.5.19 current_version: 0.5.19 release: 0.5.19 copyright_year: 2015-2019 +moban_command: false dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 From 0fe3a5d8b4bdf3700e039c0ebc8f9f7856ed056f Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 14 Jul 2019 08:32:26 +0100 Subject: [PATCH 029/143] :sparkles: add flake8 install back --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 734bd84..c560156 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,7 @@ stages: submodules: false python: 3.6 stage: lint + install: pip install flake8 script: make lint jobs: From c21d36f4d1fa26974a0d6fd78ffe906371b7c81a Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 14 Jul 2019 08:36:23 +0100 Subject: [PATCH 030/143] :green_heart: fix setup.py --- setup.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index bb0987f..3a771e4 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,8 @@ VERSION = "0.5.19" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( - "A python library to read and write structured data in csv, zipped csv" + - "format and to/from databases" + "A python library to read and write structured data in csv, zipped csv" + + "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" DOWNLOAD_URL = "%s/archive/0.5.19.tar.gz" % URL @@ -60,37 +60,38 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: Implementation :: PyPy' + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: Implementation :: PyPy", ] -INSTALL_REQUIRES = [ - "lml>=0.0.4", -] +INSTALL_REQUIRES = ["lml>=0.0.4"] SETUP_COMMANDS = {} if PY26: - INSTALL_REQUIRES.append('ordereddict') + INSTALL_REQUIRES.append("ordereddict") PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests"]) EXTRAS_REQUIRE = { - "xls": ['pyexcel-xls>=0.5.0'], - "xlsx": ['pyexcel-xlsx>=0.5.0'], - "ods": ['pyexcel-ods3>=0.5.0'], + "xls": ["pyexcel-xls>=0.5.0"], + "xlsx": ["pyexcel-xlsx>=0.5.0"], + "ods": ["pyexcel-ods3>=0.5.0"], } # You do not need to read beyond this line -PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs pyexcel-io v0.5.19 " + - "Find 0.5.19 in changelog for more details") -NO_GS_MESSAGE = ("Automatic github release is disabled. " + - "Please install gease to enable it.") +PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format( + sys.executable +) +GS_COMMAND = ( + "gs pyexcel-io v0.5.19 " + "Find 0.5.19 in changelog for more details" +) +NO_GS_MESSAGE = ( + "Automatic github release is disabled. " + + "Please install gease to enable it." +) UPLOAD_FAILED_MSG = ( - 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) + 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND +) HERE = os.path.abspath(os.path.dirname(__file__)) @@ -133,9 +134,7 @@ class PublishCommand(Command): sys.exit() -SETUP_COMMANDS.update({ - "publish": PublishCommand -}) +SETUP_COMMANDS.update({"publish": PublishCommand}) def has_gease(): @@ -146,6 +145,7 @@ def has_gease(): """ try: import gease # noqa + return True except ImportError: return False @@ -213,5 +213,5 @@ if __name__ == "__main__": include_package_data=True, zip_safe=False, classifiers=CLASSIFIERS, - cmdclass=SETUP_COMMANDS + cmdclass=SETUP_COMMANDS, ) From e65e939c1b9b04b6277a749a7fb38c9fe04932e2 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 17 Jul 2019 21:06:24 +0100 Subject: [PATCH 031/143] :bug: :lipstick: better error handling. fix #70 --- pyexcel_io/io.py | 7 ++++++- tests/test_io.py | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 136212f..e5d4fa8 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -190,7 +190,12 @@ def load_data( reader = READERS.get_a_plugin(file_type, library) except NoSupportingPluginFound: if file_name: - if not os.path.exists(file_name): + if os.path.exists(file_name): + if os.path.isfile(file_name): + raise + else: + raise IOError("%s is not a file" % file_name) + else: raise IOError("%s does not exist" % file_name) else: raise diff --git a/tests/test_io.py b/tests/test_io.py index cca6af0..a5891e2 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -15,6 +15,11 @@ from zipfile import BadZipfile PY2 = sys.version_info[0] == 2 +@raises(IOError) +def test_directory_name_as_file(): + get_data('/') + + def test_force_file_type(): test_file = "force_file_type.txt" data = get_data( From a85249530d7b944b8e6602b55b7a14119b787b02 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 17 Jul 2019 21:13:37 +0100 Subject: [PATCH 032/143] :egg: :ferris_wheel: reglease 0.5.20 --- .moban.yml | 1 + .travis.yml | 11 +++++++++- CHANGELOG.rst | 9 ++++++++ changelog.yml | 6 ++++++ docs/source/conf.py | 4 ++-- lint.sh | 1 + pyexcel-io.yml | 7 +++--- setup.py | 52 ++++++++++++++++++++++----------------------- 8 files changed, 58 insertions(+), 33 deletions(-) diff --git a/.moban.yml b/.moban.yml index 529a249..c8a761c 100644 --- a/.moban.yml +++ b/.moban.yml @@ -1,6 +1,7 @@ requires: - type: git url: https://github.com/moremoban/pypi-mobans + branch: dev submodule: true - https://github.com/pyexcel/pyexcel-mobans configuration: diff --git a/.travis.yml b/.travis.yml index c560156..51c299b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ python: stages: - test - lint + - moban .disable_global: &disable_global addons: false @@ -36,9 +37,17 @@ stages: submodules: false python: 3.6 stage: lint - install: pip install flake8 script: make lint +.moban: &moban + <<: *disable_global + python: 3.6 + stage: moban + install: pip install moban>=0.0.4 + script: + - moban + - git diff --exit-code + jobs: include: - *moban diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 60f7687..0b5e8c2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.5.20 - 17.7.2019 +-------------------------------------------------------------------------------- + +updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#70 `_: when the given file + is a root directory, the error shall read it is not a file + 0.5.19 - 14.7.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index d26e841..8dc5b05 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - '`#70`: when the given file is a root directory, the error shall read it is not a file' + version: 0.5.20 + date: 17.7.2019 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index bd43399..a847d26 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,9 +26,9 @@ project = 'pyexcel-io' copyright = '2015-2019 Onni Software Ltd.' author = 'C.W.' # The short X.Y version -version = '0.5.19' +version = '0.5.20' # The full version, including alpha/beta/rc tags -release = '0.5.19' +release = '0.5.20' # -- General configuration --------------------------------------------------- diff --git a/lint.sh b/lint.sh index 12183c3..59cda13 100644 --- a/lint.sh +++ b/lint.sh @@ -1 +1,2 @@ +pip install flake8 flake8 . --exclude=.moban.d,docs --builtins=unicode,xrange,long \ No newline at end of file diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 9cd365c..1b270ce 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,11 +2,10 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.5.19 -current_version: 0.5.19 -release: 0.5.19 +version: 0.5.20 +current_version: 0.5.20 +release: 0.5.20 copyright_year: 2015-2019 -moban_command: false dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 diff --git a/setup.py b/setup.py index 3a771e4..c60019e 100644 --- a/setup.py +++ b/setup.py @@ -29,15 +29,15 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "C.W." -VERSION = "0.5.19" +VERSION = "0.5.20" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( - "A python library to read and write structured data in csv, zipped csv" - + "format and to/from databases" + "A python library to read and write structured data in csv, zipped csv" + + "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.5.19.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.5.20.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -60,38 +60,37 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: Implementation :: PyPy", + + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python :: Implementation :: PyPy' ] -INSTALL_REQUIRES = ["lml>=0.0.4"] +INSTALL_REQUIRES = [ + "lml>=0.0.4", +] SETUP_COMMANDS = {} if PY26: - INSTALL_REQUIRES.append("ordereddict") + INSTALL_REQUIRES.append('ordereddict') PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests"]) EXTRAS_REQUIRE = { - "xls": ["pyexcel-xls>=0.5.0"], - "xlsx": ["pyexcel-xlsx>=0.5.0"], - "ods": ["pyexcel-ods3>=0.5.0"], + "xls": ['pyexcel-xls>=0.5.0'], + "xlsx": ['pyexcel-xlsx>=0.5.0'], + "ods": ['pyexcel-ods3>=0.5.0'], } # You do not need to read beyond this line -PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format( - sys.executable -) -GS_COMMAND = ( - "gs pyexcel-io v0.5.19 " + "Find 0.5.19 in changelog for more details" -) -NO_GS_MESSAGE = ( - "Automatic github release is disabled. " - + "Please install gease to enable it." -) +PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) +GS_COMMAND = ("gs pyexcel-io v0.5.20 " + + "Find 0.5.20 in changelog for more details") +NO_GS_MESSAGE = ("Automatic github release is disabled. " + + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( - 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND -) + 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) HERE = os.path.abspath(os.path.dirname(__file__)) @@ -134,7 +133,9 @@ class PublishCommand(Command): sys.exit() -SETUP_COMMANDS.update({"publish": PublishCommand}) +SETUP_COMMANDS.update({ + "publish": PublishCommand +}) def has_gease(): @@ -145,7 +146,6 @@ def has_gease(): """ try: import gease # noqa - return True except ImportError: return False @@ -213,5 +213,5 @@ if __name__ == "__main__": include_package_data=True, zip_safe=False, classifiers=CLASSIFIERS, - cmdclass=SETUP_COMMANDS, + cmdclass=SETUP_COMMANDS ) From 01ba94c3015a205f4f12383974faef5e95daea9f Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 17 Jul 2019 21:33:31 +0100 Subject: [PATCH 033/143] :fire: remove moban stage check --- .travis.yml | 10 ---------- pyexcel-io.yml | 1 + setup.py | 48 ++++++++++++++++++++++++------------------------ 3 files changed, 25 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 51c299b..734bd84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ python: stages: - test - lint - - moban .disable_global: &disable_global addons: false @@ -39,15 +38,6 @@ stages: stage: lint script: make lint -.moban: &moban - <<: *disable_global - python: 3.6 - stage: moban - install: pip install moban>=0.0.4 - script: - - moban - - git diff --exit-code - jobs: include: - *moban diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 1b270ce..e5e03c7 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -5,6 +5,7 @@ nick_name: io version: 0.5.20 current_version: 0.5.20 release: 0.5.20 +moban_command: false copyright_year: 2015-2019 dependencies: - ordereddict;python_version<"2.7" diff --git a/setup.py b/setup.py index c60019e..2938755 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,8 @@ VERSION = "0.5.20" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( - "A python library to read and write structured data in csv, zipped csv" + - "format and to/from databases" + "A python library to read and write structured data in csv, zipped csv" + + "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" DOWNLOAD_URL = "%s/archive/0.5.20.tar.gz" % URL @@ -60,37 +60,38 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: Implementation :: PyPy' + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: Implementation :: PyPy", ] -INSTALL_REQUIRES = [ - "lml>=0.0.4", -] +INSTALL_REQUIRES = ["lml>=0.0.4"] SETUP_COMMANDS = {} if PY26: - INSTALL_REQUIRES.append('ordereddict') + INSTALL_REQUIRES.append("ordereddict") PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests"]) EXTRAS_REQUIRE = { - "xls": ['pyexcel-xls>=0.5.0'], - "xlsx": ['pyexcel-xlsx>=0.5.0'], - "ods": ['pyexcel-ods3>=0.5.0'], + "xls": ["pyexcel-xls>=0.5.0"], + "xlsx": ["pyexcel-xlsx>=0.5.0"], + "ods": ["pyexcel-ods3>=0.5.0"], } # You do not need to read beyond this line -PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs pyexcel-io v0.5.20 " + - "Find 0.5.20 in changelog for more details") -NO_GS_MESSAGE = ("Automatic github release is disabled. " + - "Please install gease to enable it.") +PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format( + sys.executable +) +GS_COMMAND = ( + "gs pyexcel-io v0.5.20 " + "Find 0.5.20 in changelog for more details" +) +NO_GS_MESSAGE = ( + "Automatic github release is disabled. " + + "Please install gease to enable it." +) UPLOAD_FAILED_MSG = ( - 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) + 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND +) HERE = os.path.abspath(os.path.dirname(__file__)) @@ -133,9 +134,7 @@ class PublishCommand(Command): sys.exit() -SETUP_COMMANDS.update({ - "publish": PublishCommand -}) +SETUP_COMMANDS.update({"publish": PublishCommand}) def has_gease(): @@ -146,6 +145,7 @@ def has_gease(): """ try: import gease # noqa + return True except ImportError: return False @@ -213,5 +213,5 @@ if __name__ == "__main__": include_package_data=True, zip_safe=False, classifiers=CLASSIFIERS, - cmdclass=SETUP_COMMANDS + cmdclass=SETUP_COMMANDS, ) From 101de35f10d9d35fa13a1e55a13fd30ec9cfa6f2 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 2 Oct 2019 19:22:06 +0100 Subject: [PATCH 034/143] :books: :bug: fix the merge problem when using moban 0.6.3 --- .moban.d/{README.rst => io_readme.rst.jj2} | 0 .moban.d/{setup.py => io_setup.py.jj2} | 0 .moban.yml | 16 +- README.rst | 222 ++++++++++++++++++++- setup.py | 219 +++++++++++++++++++- 5 files changed, 436 insertions(+), 21 deletions(-) rename .moban.d/{README.rst => io_readme.rst.jj2} (100%) rename .moban.d/{setup.py => io_setup.py.jj2} (100%) diff --git a/.moban.d/README.rst b/.moban.d/io_readme.rst.jj2 similarity index 100% rename from .moban.d/README.rst rename to .moban.d/io_readme.rst.jj2 diff --git a/.moban.d/setup.py b/.moban.d/io_setup.py.jj2 similarity index 100% rename from .moban.d/setup.py rename to .moban.d/io_setup.py.jj2 diff --git a/.moban.yml b/.moban.yml index b99d891..9eb467a 100644 --- a/.moban.yml +++ b/.moban.yml @@ -1,26 +1,20 @@ -requires: - - type: git - url: https://github.com/moremoban/pypi-mobans - branch: dev - submodule: true - - https://github.com/pyexcel/pyexcel-mobans configuration: - configuration_dir: "pyexcel-mobans:config" + configuration_dir: "git://github.com/pyexcel/pyexcel-mobans!/config" template_dir: - - "pyexcel-mobans:templates" - - "pypi-mobans:templates" + - "git://github.com/moremoban/pypi-mobans.git?branch=dev&submodule=true!/templates" + - "git://github.com/pyexcel/pyexcel-mobans.git!/templates" - ".moban.d" configuration: pyexcel-io.yml targets: - "docs/source/conf.py": "docs/conf.py_t" - - setup.py: setup.py + - setup.py: io_setup.py.jj2 - .travis.yml: travis.yml.jj2 - requirements.txt: requirements.txt.jj2 - "tests/requirements.txt": "tests/requirements.txt" - LICENSE: NEW_BSD_LICENSE.jj2 - test.sh: test.script.jj2 - test.bat: test.script.jj2 - - README.rst: README.rst + - README.rst: io_readme.rst.jj2 - "docs/source/index.rst": "docs/source/index.rst" - output: CHANGELOG.rst configuration: changelog.yml diff --git a/README.rst b/README.rst index 1bc9c94..8c3f717 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,124 @@ -{%extends "BASIC-README.rst.jj2"%} +================================================================================ +pyexcel-io - Let you focus on data, instead of file formats +================================================================================ -{%block features%} +.. image:: https://raw.githubusercontent.com/pyexcel/pyexcel.github.io/master/images/patreon.png + :target: https://www.patreon.com/chfw -**{{name}}** provides **one** application programming interface(API) to read +.. image:: https://api.bountysource.com/badge/team?team_id=288537 + :target: https://salt.bountysource.com/teams/chfw-pyexcel + +.. image:: https://travis-ci.org/pyexcel/pyexcel-io.svg?branch=master + :target: http://travis-ci.org/pyexcel/pyexcel-io + +.. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg + :target: https://codecov.io/gh/pyexcel/pyexcel-io + +.. image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg + :target: https://gitter.im/pyexcel/Lobby + +.. image:: https://readthedocs.org/projects/pyexcel-io/badge/?version=latest + :target: http://pyexcel-io.readthedocs.org/en/latest/ + +Support the project +================================================================================ + +If your company has embedded pyexcel and its components into a revenue generating +product, please support me on `patreon `_ +or `bounty source `_ to maintain +the project and develop it further. + +If you are an individual, you are welcome to support me too and for however long +you feel like. As my backer, you will receive +`early access to pyexcel related contents `_. + +And your issues will get prioritized if you would like to become my patreon as `pyexcel pro user`. + +With your financial support, I will be able to invest +a little bit more time in coding, documentation and writing interesting posts. + + +Known constraints +================== + +Fonts, colors and charts are not supported. + +Introduction +================================================================================ + +**pyexcel-io** provides **one** application programming interface(API) to read and write the data in excel format, import the data into and export the data from database. It provides support for csv(z) format, django database and sqlalchemy supported databases. Its supported file formats are extended to cover "xls", "xlsx", "ods" by the following extensions: -{% include "io-plugins-list.rst.jj2" %} +.. _file-format-list: +.. _a-map-of-plugins-and-file-formats: + +.. table:: A list of file formats supported by external plugins + + ======================== ======================= ================= ================== + Package name Supported file formats Dependencies Python versions + ======================== ======================= ================= ================== + `pyexcel-io`_ csv, csvz [#f1]_, tsv, 2.6, 2.7, 3.3, + tsvz [#f2]_ 3.4, 3.5, 3.6 + pypy + `pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, same as above + xlsm(read only) `xlwt`_ + `pyexcel-xlsx`_ xlsx `openpyxl`_ same as above + `pyexcel-ods3`_ ods `pyexcel-ezodf`_, 2.6, 2.7, 3.3, 3.4 + lxml 3.5, 3.6 + `pyexcel-ods`_ ods `odfpy`_ same as above + ======================== ======================= ================= ================== + +.. table:: Dedicated file reader and writers + + ======================== ======================= ================= ================== + Package name Supported file formats Dependencies Python versions + ======================== ======================= ================= ================== + `pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ Python 2 and 3 + `pyexcel-xlsxr`_ xlsx(read only) lxml same as above + `pyexcel-xlsbr`_ xlsx(read only) pyxlsb same as above + `pyexcel-odsr`_ read only for ods, fods lxml same as above + `pyexcel-odsw`_ write only for ods loxun same as above + `pyexcel-htmlr`_ html(read only) lxml,html5lib same as above + `pyexcel-pdfr`_ pdf(read only) pdftables Python 2 only. + ======================== ======================= ================= ================== + + +.. _pyexcel-io: https://github.com/pyexcel/pyexcel-io +.. _pyexcel-xls: https://github.com/pyexcel/pyexcel-xls +.. _pyexcel-xlsx: https://github.com/pyexcel/pyexcel-xlsx +.. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods +.. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3 +.. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr +.. _pyexcel-odsw: https://github.com/pyexcel/pyexcel-odsw +.. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr + +.. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw +.. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr +.. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr +.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr + +.. _xlrd: https://github.com/python-excel/xlrd +.. _xlwt: https://github.com/python-excel/xlwt +.. _openpyxl: https://bitbucket.org/openpyxl/openpyxl +.. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter +.. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf +.. _odfpy: https://github.com/eea/odfpy + + +In order to manage the list of plugins installed, you need to use pip to add or remove +a plugin. When you use virtualenv, you can have different plugins per virtual +environment. In the situation where you have multiple plugins that does the same thing +in your environment, you need to tell pyexcel which plugin to use per function call. +For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr. +You need to append get_array(..., library='pyexcel-odsr'). + +.. rubric:: Footnotes + +.. [#f1] zipped csv file +.. [#f2] zipped tsv file If you need to manipulate the data, you might do it yourself or use its brother library `pyexcel `__ . @@ -16,4 +126,106 @@ library `pyexcel `__ . If you would like to extend it, you may use it to write your own extension to handle a specific file format. -{%endblock%} + + + +Installation +================================================================================ + +You can install pyexcel-io via pip: + +.. code-block:: bash + + $ pip install pyexcel-io + + +or clone it and install it: + +.. code-block:: bash + + $ git clone https://github.com/pyexcel/pyexcel-io.git + $ cd pyexcel-io + $ python setup.py install + + + +Development guide +================================================================================ + +Development steps for code changes + +#. git clone https://github.com/pyexcel/pyexcel-io.git +#. cd pyexcel-io + +Upgrade your setup tools and pip. They are needed for development and testing only: + +#. pip install --upgrade setuptools pip + +Then install relevant development requirements: + +#. pip install -r rnd_requirements.txt # if such a file exists +#. pip install -r requirements.txt +#. pip install -r tests/requirements.txt + +Once you have finished your changes, please provide test case(s), relevant documentation +and update CHANGELOG.rst. + +.. note:: + + As to rnd_requirements.txt, usually, it is created when a dependent + library is not released. Once the dependecy is installed + (will be released), the future + version of the dependency in the requirements.txt will be valid. + + +How to test your contribution +------------------------------ + +Although `nose` and `doctest` are both used in code testing, it is adviable that unit tests are put in tests. `doctest` is incorporated only to make sure the code examples in documentation remain valid across different development releases. + +On Linux/Unix systems, please launch your tests like this:: + + $ make + +On Windows systems, please issue this command:: + + > test.bat + +How to update test environment and update documentation +--------------------------------------------------------- + +Additional steps are required: + +#. pip install moban +#. git clone https://github.com/moremoban/setupmobans.git # generic setup +#. git clone https://github.com/pyexcel/pyexcel-commons.git commons +#. make your changes in `.moban.d` directory, then issue command `moban` + +What is pyexcel-commons +--------------------------------- + +Many information that are shared across pyexcel projects, such as: this developer guide, license info, etc. are stored in `pyexcel-commons` project. + +What is .moban.d +--------------------------------- + +`.moban.d` stores the specific meta data for the library. + +Acceptance criteria +------------------- + +#. Has Test cases written +#. Has all code lines tested +#. Passes all Travis CI builds +#. Has fair amount of documentation if your change is complex +#. run 'make format' so as to confirm the pyexcel organisation's coding style +#. Please update CHANGELOG.rst +#. Please add yourself to CONTRIBUTORS.rst +#. Agree on NEW BSD License for your contribution + + + +License +================================================================================ + +New BSD License diff --git a/setup.py b/setup.py index bf85899..9fed34c 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,217 @@ -{% extends 'pyexcel-setup.py.jj2' %} +#!/usr/bin/env python3 -{%block platform_block%} -{%endblock%} +# Template by pypi-mobans +import codecs +import locale +import os +import platform +import sys +from shutil import rmtree -{%block pyexcel_extra_classifiers%} +from setuptools import Command, find_packages, setup + +PY2 = sys.version_info[0] == 2 +PY26 = PY2 and sys.version_info[1] < 7 +PY33 = sys.version_info < (3, 4) + +# Work around mbcs bug in distutils. +# http://bugs.python.org/issue10945 +# This work around is only if a project supports Python < 3.4 + +# Work around for locale not being set +try: + lc = locale.getlocale() + pf = platform.system() + if pf != "Windows" and lc == (None, None): + locale.setlocale(locale.LC_ALL, "C.UTF-8") +except (ValueError, UnicodeError, locale.Error): + locale.setlocale(locale.LC_ALL, "en_US.UTF-8") + +NAME = "pyexcel-io" +AUTHOR = "C.W." +VERSION = "0.6.0" +EMAIL = "info@pyexcel.org" +LICENSE = "New BSD" +DESCRIPTION = ( + "A python library to read and write structured data in csv, zipped csv" + + "format and to/from databases" +) +URL = "https://github.com/pyexcel/pyexcel-io" +DOWNLOAD_URL = "%s/archive/0.5.20.tar.gz" % URL +FILES = ["README.rst", "CHANGELOG.rst"] +KEYWORDS = [ + "python", + "API", + "tsv", + "tsvz", + "csv", + "csvz", + "django", + "sqlalchemy", +] + +CLASSIFIERS = [ + "Topic :: Software Development :: Libraries", + "Programming Language :: Python", + "Intended Audience :: Developers", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + + "Programming Language :: Python :: 3.7", + + "Programming Language :: Python :: 3.8", + + 'License :: OSI Approved :: BSD License', 'Programming Language :: Python :: Implementation :: PyPy' -{%endblock%}} +] + +INSTALL_REQUIRES = [ + "lml>=0.0.4", +] +SETUP_COMMANDS = {} + +if PY26: + INSTALL_REQUIRES.append('ordereddict') + +PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests"]) +EXTRAS_REQUIRE = { + "xls": ['pyexcel-xls>=0.5.0'], + "xlsx": ['pyexcel-xlsx>=0.5.0'], + "ods": ['pyexcel-ods3>=0.5.0'], +} +# You do not need to read beyond this line +PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) +GS_COMMAND = ("gs pyexcel-io v0.5.20 " + + "Find 0.5.20 in changelog for more details") +NO_GS_MESSAGE = ("Automatic github release is disabled. " + + "Please install gease to enable it.") +UPLOAD_FAILED_MSG = ( + 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) +HERE = os.path.abspath(os.path.dirname(__file__)) + + +class PublishCommand(Command): + """Support setup.py upload.""" + + description = "Build and publish the package on github and pypi" + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print("\033[1m{0}\033[0m".format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status("Removing previous builds...") + rmtree(os.path.join(HERE, "dist")) + rmtree(os.path.join(HERE, "build")) + rmtree(os.path.join(HERE, "pyexcel_io.egg-info")) + except OSError: + pass + + self.status("Building Source and Wheel (universal) distribution...") + run_status = True + if has_gease(): + run_status = os.system(GS_COMMAND) == 0 + else: + self.status(NO_GS_MESSAGE) + if run_status: + if os.system(PUBLISH_COMMAND) != 0: + self.status(UPLOAD_FAILED_MSG % PUBLISH_COMMAND) + + sys.exit() + + +SETUP_COMMANDS.update({ + "publish": PublishCommand +}) + + +def has_gease(): + """ + test if github release command is installed + + visit http://github.com/moremoban/gease for more info + """ + try: + import gease # noqa + return True + except ImportError: + return False + + +def read_files(*files): + """Read files into setup""" + text = "" + for single_file in files: + content = read(single_file) + text = text + content + "\n" + return text + + +def read(afile): + """Read a file into setup""" + the_relative_file = os.path.join(HERE, afile) + with codecs.open(the_relative_file, "r", "utf-8") as opened_file: + content = filter_out_test_code(opened_file) + content = "".join(list(content)) + return content + + +def filter_out_test_code(file_handle): + found_test_code = False + for line in file_handle.readlines(): + if line.startswith(".. testcode:"): + found_test_code = True + continue + if found_test_code is True: + if line.startswith(" "): + continue + else: + empty_line = line.strip() + if len(empty_line) == 0: + continue + else: + found_test_code = False + yield line + else: + for keyword in ["|version|", "|today|"]: + if keyword in line: + break + else: + yield line + + +if __name__ == "__main__": + setup( + test_suite="tests", + name=NAME, + author=AUTHOR, + version=VERSION, + author_email=EMAIL, + description=DESCRIPTION, + url=URL, + download_url=DOWNLOAD_URL, + long_description=read_files(*FILES), + license=LICENSE, + keywords=KEYWORDS, + extras_require=EXTRAS_REQUIRE, + tests_require=["nose"], + install_requires=INSTALL_REQUIRES, + packages=PACKAGES, + include_package_data=True, + zip_safe=False, + classifiers=CLASSIFIERS, + cmdclass=SETUP_COMMANDS + ) From 8206b2d8037a54f4d9f9761050333b06c5479407 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 2 Oct 2019 19:32:02 +0100 Subject: [PATCH 035/143] :books: add sponsor logo --- .github/FUNDING.yml | 3 +++ .moban.yml | 2 ++ README.rst | 2 -- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..edd187d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +patreon: pyexcel diff --git a/.moban.yml b/.moban.yml index 9eb467a..dd021c5 100644 --- a/.moban.yml +++ b/.moban.yml @@ -3,6 +3,7 @@ configuration: template_dir: - "git://github.com/moremoban/pypi-mobans.git?branch=dev&submodule=true!/templates" - "git://github.com/pyexcel/pyexcel-mobans.git!/templates" + - "git://github.com/pyexcel/pyexcel-mobans.git!/statics" - ".moban.d" configuration: pyexcel-io.yml targets: @@ -20,3 +21,4 @@ targets: configuration: changelog.yml template: CHANGELOG.rst.jj2 - lint.sh: lint.script.jj2 + - ".github/FUNDING.yml": "FUNDING.yml" diff --git a/README.rst b/README.rst index 8c3f717..11a4177 100644 --- a/README.rst +++ b/README.rst @@ -197,8 +197,6 @@ How to update test environment and update documentation Additional steps are required: #. pip install moban -#. git clone https://github.com/moremoban/setupmobans.git # generic setup -#. git clone https://github.com/pyexcel/pyexcel-commons.git commons #. make your changes in `.moban.d` directory, then issue command `moban` What is pyexcel-commons From 89f53f51a42a61d31ffb336a2a27402fc64988dd Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 5 Oct 2019 12:32:48 +0100 Subject: [PATCH 036/143] :books: update funding link --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index edd187d..7f9d74f 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,3 @@ # These are supported funding model platforms -patreon: pyexcel +patreon: chfw From 12f12346cb4e2f71d1250756674516ca528615ea Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 5 Oct 2019 12:43:41 +0100 Subject: [PATCH 037/143] :books: update PR guide --- .github/PULL_REQUEST_TEMPLATE.md | 10 ++++++++++ .moban.yml | 1 + README.rst | 12 ------------ 3 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..7b632ce --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,10 @@ +With your PR, here is a check list: + +- [ ] Has Test cases written +- [ ] Has all code lines tested +- [ ] Passes all Travis CI builds +- [ ] Has fair amount of documentation if your change is complex +- [ ] run 'make format' so as to confirm the pyexcel organisation's coding style +- [ ] Please update CHANGELOG.rst +- [ ] Please add yourself to CONTRIBUTORS.rst +- [ ] Agree on NEW BSD License for your contribution diff --git a/.moban.yml b/.moban.yml index dd021c5..0eadef5 100644 --- a/.moban.yml +++ b/.moban.yml @@ -22,3 +22,4 @@ targets: template: CHANGELOG.rst.jj2 - lint.sh: lint.script.jj2 - ".github/FUNDING.yml": "FUNDING.yml" + - ".github/PULL_REQUEST_TEMPLATE.md": PULL_REQUEST_TEMPLATE.md diff --git a/README.rst b/README.rst index 11a4177..f5fbd3b 100644 --- a/README.rst +++ b/README.rst @@ -209,18 +209,6 @@ What is .moban.d `.moban.d` stores the specific meta data for the library. -Acceptance criteria -------------------- - -#. Has Test cases written -#. Has all code lines tested -#. Passes all Travis CI builds -#. Has fair amount of documentation if your change is complex -#. run 'make format' so as to confirm the pyexcel organisation's coding style -#. Please update CHANGELOG.rst -#. Please add yourself to CONTRIBUTORS.rst -#. Agree on NEW BSD License for your contribution - License From 3a11f43f10f39fff188b0815493012ac5c690a80 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 21 Oct 2019 22:25:09 +0100 Subject: [PATCH 038/143] :handshake: sync with pyexcel-mobans and pypi-mobans --- .github/FUNDING.yml | 1 + CHANGELOG.rst | 135 +++++++++++++++----------------------------- docs/source/conf.py | 4 +- setup.py | 2 +- 4 files changed, 49 insertions(+), 93 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 7f9d74f..0faea60 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,4 @@ # These are supported funding model platforms +github: chfw patreon: chfw diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0b5e8c2..dbb6d48 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,8 +4,7 @@ Change log 0.5.20 - 17.7.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#70 `_: when the given file is a root directory, the error shall read it is not a file @@ -13,8 +12,7 @@ updated 0.5.19 - 14.7.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `pyexcel#185 `_: handle stream conversion if file type(html) needs string content then bytes to handle @@ -22,8 +20,7 @@ updated 0.5.18 - 12.06.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#69 `_: Force file type(force_file_type) on write @@ -31,8 +28,7 @@ updated 0.5.17 - 04.04.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#68 `_: Raise IOError when the data file does not exist @@ -40,8 +36,7 @@ updated 0.5.16 - 19.03.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#67 `_: fix conversion issue for long type on python 2.7 for ods @@ -49,8 +44,7 @@ updated 0.5.15 - 16.03.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `pyexcel-ods#33 `_: fix integer comparision error on i586 @@ -58,8 +52,7 @@ updated 0.5.14 - 21.02.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#65 `_: add tests/__init__.py because python2.7 setup.py test needs it @@ -67,8 +60,7 @@ updated 0.5.13 - 12.02.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#63 `_: Version 0.5.12 prevents xslx and ods plugin from being loaded @@ -76,8 +68,7 @@ updated 0.5.12 - 9.02.2019 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#60 `_: include tests in tar ball @@ -87,8 +78,7 @@ updated 0.5.11 - 3.12.2018 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#59 `_: Please use scan_plugins_regex, which lml 0.7 complains about @@ -96,8 +86,7 @@ updated 0.5.10 - 27.11.2018 -------------------------------------------------------------------------------- -added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**added** #. `#57 `_, long type will not be written in ods. please use string type. And if the integer is equal or @@ -108,8 +97,7 @@ added 0.5.9.1 - 30.08.2018 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. `#53 `_, upgrade lml dependency to at least 0.0.2 @@ -117,8 +105,7 @@ updated 0.5.9 - 23.08.2018 -------------------------------------------------------------------------------- -added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**added** #. `pyexcel#148 `_, support force_file_type @@ -126,8 +113,7 @@ added 0.5.8 - 16.08.2018 -------------------------------------------------------------------------------- -added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**added** #. `#49 `_, support additional options when detecting float values in csv format. default_float_nan, @@ -136,8 +122,7 @@ added 0.5.7 - 02.05.2018 -------------------------------------------------------------------------------- -fixed -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**fixed** #. `#48 `_, turn off pep 0515 #. `#47 `_, csv reader cannot @@ -146,8 +131,7 @@ fixed 0.5.6 - 11.01.2018 -------------------------------------------------------------------------------- -fixed -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**fixed** #. `#46 `_, expose `bulk_save` to developer @@ -155,8 +139,7 @@ fixed 0.5.5 - 23.12.2017 -------------------------------------------------------------------------------- -fixed -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**fixed** #. Issue `#45 `_, csv reader throws exception because google app engine does not support mmap. People who @@ -166,8 +149,7 @@ fixed 0.5.4 - 10.11.2017 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. PR `#44 `_, use unicodewriter for csvz writers. @@ -175,8 +157,7 @@ updated 0.5.3 - 23.10.2017 -------------------------------------------------------------------------------- -updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**updated** #. pyexcel `pyexcel#105 `_, remove gease from setup_requires, introduced by 0.5.2. @@ -185,8 +166,7 @@ updated 0.5.2 - 20.10.2017 -------------------------------------------------------------------------------- -added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**added** #. `pyexcel#103 `_, include LICENSE file in MANIFEST.in, meaning LICENSE file will appear in the released @@ -195,8 +175,7 @@ added 0.5.1 - 02.09.2017 -------------------------------------------------------------------------------- -Fixed -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Fixed** #. `pyexcel-ods#25 `_, Unwanted dependency on pyexcel. @@ -204,13 +183,11 @@ Fixed 0.5.0 - 30.08.2017 -------------------------------------------------------------------------------- -Added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Added** #. Collect all data type conversion codes as service.py. -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#19 `_, use cString by default. For python, it will be a performance boost @@ -218,8 +195,7 @@ Updated 0.4.4 - 08.08.2017 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#42 `_, raise exception if database table name does not match the sheet name @@ -227,8 +203,7 @@ Updated 0.4.3 - 29.07.2017 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#41 `_, walk away gracefully when mmap is not available. @@ -236,8 +211,7 @@ Updated 0.4.2 - 05.07.2017 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#37 `_, permanently fix the residue folder pyexcel by release all future releases in a clean clone. @@ -245,8 +219,7 @@ Updated 0.4.1 - 29.06.2017 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#39 `_, raise exception when bulk save in django fails. Please `bulk_save=False` if you as the @@ -258,8 +231,7 @@ Updated 0.4.0 - 19.06.2017 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. 'built-in' as the value to the parameter 'library' as parameter to invoke pyexcel-io's built-in csv, tsv, csvz, tsvz, django and sql won't work. It is @@ -270,16 +242,14 @@ Updated handle are made sure to be closed. File close mechanism is enfored. #. iget_data function is introduced to cope with dangling file handle issue. -Removed -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Removed** #. Removed plugin loading code and lml is used instead 0.3.4 - 18.05.2017 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#33 `_, handle mmap object differently given as file content. This issue has put in a priority to single @@ -295,16 +265,14 @@ Updated 0.3.3 - 30.03.2017 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#31 `_, support pyinstaller 0.3.2 - 26.01.2017 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#29 `_, change skip_empty_rows to False by default @@ -312,13 +280,11 @@ Updated 0.3.1 - 21.01.2017 -------------------------------------------------------------------------------- -Added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Added** #. updated versions of extra packages -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#23 `_, provide helpful message when old pyexcel plugin exists @@ -327,8 +293,7 @@ Updated 0.3.0 - 22.12.2016 -------------------------------------------------------------------------------- -Added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Added** #. lazy loading of plugins. for example, pyexcel-xls is not entirely loaded until xls format is used at its first attempted reading or writing. Since it @@ -339,16 +304,14 @@ Added 0.2.6 - 21.12.2016 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#24 `__, pass on batch_size 0.2.5 - 20.12.2016 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#26 `__, performance issue with getting the number of columns. @@ -356,8 +319,7 @@ Updated 0.2.4 - 24.11.2016 -------------------------------------------------------------------------------- -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#23 `__, Failed to convert long integer string in python 2 to its actual value @@ -365,8 +327,7 @@ Updated 0.2.3 - 16.09.2016 -------------------------------------------------------------------------------- -Added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Added** #. `#21 `__, choose subset from data base tables for export @@ -376,16 +337,14 @@ Added 0.2.2 - 31.08.2016 -------------------------------------------------------------------------------- -Added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Added** #. support pagination. two pairs: start_row, row_limit and start_column, column_limit help you deal with large files. #. `skip_empty_rows=True` was introduced. To include empty rows, put it to False. -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. `#20 `__, pyexcel-io attempts to parse cell contents of 'infinity' as a float/int, crashes @@ -393,8 +352,7 @@ Updated 0.2.1 - 11.07.2016 -------------------------------------------------------------------------------- -Added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Added** #. csv format: handle utf-16 encoded csv files. Potentially being able to decode other formats if correct "encoding" is provided @@ -402,8 +360,7 @@ Added supported #. support stdin as input stream and stdout as output stream -Updated -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Updated** #. Attention, user of pyexcel-io! No longer io stream validation is performed in python 3. The guideline is: io.StringIO for csv, tsv only, otherwise BytesIO @@ -415,8 +372,7 @@ Updated 0.2.0 - 01.06.2016 -------------------------------------------------------------------------------- -Added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Added** #. autoload of pyexcel-io plugins #. auto detect `datetime`, `float` and `int`. Detection can be switched off by @@ -425,7 +381,6 @@ Added 0.1.0 - 17.01.2016 -------------------------------------------------------------------------------- -Added -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Added** #. yield key word to return generator as content diff --git a/docs/source/conf.py b/docs/source/conf.py index 70cd912..2c75e9f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -2,7 +2,7 @@ # # This file only contains a selection of the most common options. For a full # list see the documentation: -# http://www.sphinx-doc.org/en/master/config +# https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Path setup -------------------------------------------------------------- @@ -69,4 +69,4 @@ html_static_path = ['static'] # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {'https://docs.python.org/3/': None} \ No newline at end of file diff --git a/setup.py b/setup.py index 9fed34c..0389be5 100644 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ class PublishCommand(Command): self.status(NO_GS_MESSAGE) if run_status: if os.system(PUBLISH_COMMAND) != 0: - self.status(UPLOAD_FAILED_MSG % PUBLISH_COMMAND) + self.status(UPLOAD_FAILED_MSG) sys.exit() From ade0e1070547afd8663c39796142e417c836ce15 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 3 Nov 2019 21:27:31 +0000 Subject: [PATCH 039/143] :shirt: add downloads per month statstics --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index f5fbd3b..0c5fc9b 100644 --- a/README.rst +++ b/README.rst @@ -14,6 +14,9 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg :target: https://codecov.io/gh/pyexcel/pyexcel-io +.. image:: https://pepy.tech/badge/pyexcel-io/month + :target: https://pepy.tech/project/pyexcel-io/month + .. image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg :target: https://gitter.im/pyexcel/Lobby From 7125899ee4069e2cd0f7e0dd06e619dee4cbcdec Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 3 Nov 2019 21:48:32 +0000 Subject: [PATCH 040/143] :hammer: code refactoring on exception strings --- pyexcel_io/constants.py | 3 +++ pyexcel_io/io.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pyexcel_io/constants.py b/pyexcel_io/constants.py index 24b838a..94a6549 100644 --- a/pyexcel_io/constants.py +++ b/pyexcel_io/constants.py @@ -16,6 +16,7 @@ MESSAGE_INVALID_PARAMETERS = "Invalid parameters" MESSAGE_ERROR_02 = "No content, file name. Nothing is given" MESSAGE_ERROR_03 = "cannot handle unknown content" MESSAGE_WRONG_IO_INSTANCE = "Wrong io instance is passed for your file format." +MESSAGE_FILE_NAME_SHOULD_BE_STRING = "file_name should be a string" MESSAGE_CANNOT_WRITE_STREAM_FORMATTER = ( "Cannot write content of file type %s to stream" ) @@ -31,6 +32,8 @@ MESSAGE_CANNOT_READ_FILE_TYPE_FORMATTER = ( MESSAGE_LOADING_FORMATTER = ( "The plugin for file type %s is not installed. Please install %s" ) +MESSAGE_NOT_FILE_FORMATTER = "%s is not a file" +MESSAGE_FILE_DOES_NOT_EXIST = "%s does not exist" MESSAGE_EMPTY_ARRAY = "One empty row is found" MESSAGE_IGNORE_ROW = "One row is ignored" MESSAGE_DB_EXCEPTION = """ diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 3aceac3..39fe6cf 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -168,7 +168,7 @@ def load_data( try: file_type = file_name.split(".")[-1] except AttributeError: - raise Exception("file_name should be a string type") + raise Exception(constants.MESSAGE_FILE_NAME_SHOULD_BE_STRING) try: reader = READERS.get_a_plugin(file_type, library) @@ -178,9 +178,9 @@ def load_data( if os.path.isfile(file_name): raise else: - raise IOError("%s is not a file" % file_name) + raise IOError(constants.MESSAGE_NOT_FILE_FORMATTER % file_name) else: - raise IOError("%s does not exist" % file_name) + raise IOError(constants.MESSAGE_FILE_DOES_NOT_EXIST % file_name) else: raise @@ -227,7 +227,7 @@ def get_writer( try: file_type = file_name.split(".")[-1] except AttributeError: - raise Exception("file_name should be a string type") + raise Exception(constants.MESSAGE_FILE_NAME_SHOULD_BE_STRING) file_type_given = False From d3a44fafd73ceddaa7194f24e82fd0929990145e Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 5 Nov 2019 07:49:04 +0000 Subject: [PATCH 041/143] :sparkles: update conda badges --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 0c5fc9b..7466545 100644 --- a/README.rst +++ b/README.rst @@ -14,9 +14,15 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg :target: https://codecov.io/gh/pyexcel/pyexcel-io +.. image:: https://anaconda.org/conda-forge/pyexcel-io/badges/version.svg + :target: https://anaconda.org/conda-forge/pyexcel-io + .. image:: https://pepy.tech/badge/pyexcel-io/month :target: https://pepy.tech/project/pyexcel-io/month +.. image:: https://anaconda.org/conda-forge/pyexcel-io/badges/downloads.svg + :target: https://anaconda.org/conda-forge/pyexcel-io + .. image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg :target: https://gitter.im/pyexcel/Lobby From e79bb588daace280a54d239496db99af525a63f9 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Nov 2019 20:26:42 +0000 Subject: [PATCH 042/143] :books: update sponsorship details --- README.rst | 5 ++++- pyexcel_io/constants.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 7466545..efc4581 100644 --- a/README.rst +++ b/README.rst @@ -14,6 +14,9 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg :target: https://codecov.io/gh/pyexcel/pyexcel-io +.. image:: https://badge.fury.io/py/pyexcel-io.svg + :target: https://pypi.org/project/pyexcel-io + .. image:: https://anaconda.org/conda-forge/pyexcel-io/badges/version.svg :target: https://anaconda.org/conda-forge/pyexcel-io @@ -33,7 +36,7 @@ Support the project ================================================================================ If your company has embedded pyexcel and its components into a revenue generating -product, please support me on `patreon `_ +product, please support me on `github `_, `patreon `_ or `bounty source `_ to maintain the project and develop it further. diff --git a/pyexcel_io/constants.py b/pyexcel_io/constants.py index 94a6549..f969f40 100644 --- a/pyexcel_io/constants.py +++ b/pyexcel_io/constants.py @@ -4,7 +4,7 @@ Constants appeared in pyexcel - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2019 by Onni Software Ltd. :license: New BSD License """ # flake8: noqa From 00a2872a52766abad23ccd661e431503dd957a9f Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Nov 2019 20:28:40 +0000 Subject: [PATCH 043/143] :fire: remove bounty source. the software was done so good that no bounty source offered --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index efc4581..0cc0c2f 100644 --- a/README.rst +++ b/README.rst @@ -5,9 +5,6 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://raw.githubusercontent.com/pyexcel/pyexcel.github.io/master/images/patreon.png :target: https://www.patreon.com/chfw -.. image:: https://api.bountysource.com/badge/team?team_id=288537 - :target: https://salt.bountysource.com/teams/chfw-pyexcel - .. image:: https://travis-ci.org/pyexcel/pyexcel-io.svg?branch=master :target: http://travis-ci.org/pyexcel/pyexcel-io From 4582535f0958968b46be14552d76a78ccb0f0ca6 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Nov 2019 22:45:08 +0000 Subject: [PATCH 044/143] :lipstick: update flag --- pyexcel-io.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyexcel-io.yml b/pyexcel-io.yml index ce998f1..1545a79 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -7,6 +7,7 @@ current_version: 0.6.0 release: 0.5.20 copyright_year: 2015-2019 moban_command: false +is_on_conda: true dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 From 84a5bd6ba1b46021e9b6141eb49b151e3dce8c35 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Nov 2019 23:31:49 +0000 Subject: [PATCH 045/143] :shirt: update coding style --- docs/source/conf.py | 2 +- pyexcel_io/book.py | 14 ++++++++------ pyexcel_io/io.py | 16 ++++++++++++---- setup.py | 6 +++--- tests/test_book.py | 24 ++++++++++++++---------- tests/test_io.py | 12 +++--------- 6 files changed, 41 insertions(+), 33 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2c75e9f..a461dd6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,4 +69,4 @@ html_static_path = ['static'] # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/3/': None} \ No newline at end of file +intersphinx_mapping = {'https://docs.python.org/3/': None} diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index acfb140..d728666 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -234,14 +234,16 @@ def _convert_content_to_stream(file_content, file_type): stream = manager.get_io(file_type) if not PY2: target_content_type = manager.get_io_type(file_type) - needs_encode = (target_content_type == 'bytes' and - not isinstance(file_content, bytes)) - needs_decode = (target_content_type == 'string' and - isinstance(file_content, bytes)) + needs_encode = target_content_type == "bytes" and not isinstance( + file_content, bytes + ) + needs_decode = target_content_type == "string" and isinstance( + file_content, bytes + ) if needs_encode: - file_content = file_content.encode('utf-8') + file_content = file_content.encode("utf-8") elif needs_decode: - file_content = file_content.decode('utf-8') + file_content = file_content.decode("utf-8") stream.write(file_content) stream.seek(0) return stream diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 39fe6cf..20f5f1a 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -178,9 +178,13 @@ def load_data( if os.path.isfile(file_name): raise else: - raise IOError(constants.MESSAGE_NOT_FILE_FORMATTER % file_name) + raise IOError( + constants.MESSAGE_NOT_FILE_FORMATTER % file_name + ) else: - raise IOError(constants.MESSAGE_FILE_DOES_NOT_EXIST % file_name) + raise IOError( + constants.MESSAGE_FILE_DOES_NOT_EXIST % file_name + ) else: raise @@ -208,8 +212,12 @@ def load_data( def get_writer( - file_name=None, file_stream=None, file_type=None, - library=None, force_file_type=None, **keywords + file_name=None, + file_stream=None, + file_type=None, + library=None, + force_file_type=None, + **keywords ): """find a suitable writer""" inputs = [file_name, file_stream] diff --git a/setup.py b/setup.py index 0389be5..9524edd 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 +import os +import sys # Template by pypi-mobans import codecs import locale -import os import platform -import sys from shutil import rmtree -from setuptools import Command, find_packages, setup +from setuptools import Command, setup, find_packages PY2 = sys.version_info[0] == 2 PY26 = PY2 and sys.version_info[1] < 7 diff --git a/tests/test_book.py b/tests/test_book.py index cff86ba..14755a2 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -1,9 +1,13 @@ -from pyexcel_io.book import BookReader, BookWriter, RWInterface +from pyexcel_io.book import ( + BookReader, + BookWriter, + RWInterface, + _convert_content_to_stream, +) +from pyexcel_io._compact import PY2, BytesIO, StringIO -from nose.tools import raises -from pyexcel_io.book import _convert_content_to_stream -from pyexcel_io._compact import PY2, StringIO, BytesIO from nose import SkipTest +from nose.tools import raises @raises(NotImplementedError) @@ -38,17 +42,17 @@ def test_book_writer(): def test_convert_to_bytes_stream(): if PY2: - raise SkipTest('No need test in python 2') + raise SkipTest("No need test in python 2") else: - file_content = b'test' - stream = _convert_content_to_stream(file_content, 'csv') + file_content = b"test" + stream = _convert_content_to_stream(file_content, "csv") assert isinstance(stream, StringIO) def test_convert_to_string_stream(): if PY2: - raise SkipTest('No need test in python 2') + raise SkipTest("No need test in python 2") else: - file_content = 'test' - stream = _convert_content_to_stream(file_content, 'csvz') + file_content = "test" + stream = _convert_content_to_stream(file_content, "csvz") assert isinstance(stream, BytesIO) diff --git a/tests/test_io.py b/tests/test_io.py index d2ed009..1870c74 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -17,7 +17,7 @@ PY2 = sys.version_info[0] == 2 @raises(IOError) def test_directory_name_as_file(): - get_data('/') + get_data("/") def test_force_file_type(): @@ -31,14 +31,8 @@ def test_force_file_type(): def test_force_file_type_on_write(): test_file = "force_file_type_on_write.txt" - save_data( - test_file, - {"sheet 1": [[1, 2]]}, - force_file_type="csv" - ) - data = get_data( - test_file, force_file_type="csv" - ) + save_data(test_file, {"sheet 1": [[1, 2]]}, force_file_type="csv") + data = get_data(test_file, force_file_type="csv") expected = [[1, 2]] eq_(expected, data[test_file]) os.unlink(test_file) From 24a5f35b4a872184e9b7d31fda3969c2f1af7c09 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Nov 2019 23:36:59 +0000 Subject: [PATCH 046/143] :bug: update test requirements.txt --- .../{requirements.txt => custom-requirements.txt.jj2} | 0 .moban.yml | 2 +- docs/source/conf.py | 2 +- setup.py | 6 +++--- tests/requirements.txt | 8 +++++--- 5 files changed, 10 insertions(+), 8 deletions(-) rename .moban.d/tests/{requirements.txt => custom-requirements.txt.jj2} (100%) diff --git a/.moban.d/tests/requirements.txt b/.moban.d/tests/custom-requirements.txt.jj2 similarity index 100% rename from .moban.d/tests/requirements.txt rename to .moban.d/tests/custom-requirements.txt.jj2 diff --git a/.moban.yml b/.moban.yml index 0eadef5..5dd47b3 100644 --- a/.moban.yml +++ b/.moban.yml @@ -11,7 +11,7 @@ targets: - setup.py: io_setup.py.jj2 - .travis.yml: travis.yml.jj2 - requirements.txt: requirements.txt.jj2 - - "tests/requirements.txt": "tests/requirements.txt" + - "tests/requirements.txt": "tests/custom-requirements.txt.jj2" - LICENSE: NEW_BSD_LICENSE.jj2 - test.sh: test.script.jj2 - test.bat: test.script.jj2 diff --git a/docs/source/conf.py b/docs/source/conf.py index a461dd6..2c75e9f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,4 +69,4 @@ html_static_path = ['static'] # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/3/': None} +intersphinx_mapping = {'https://docs.python.org/3/': None} \ No newline at end of file diff --git a/setup.py b/setup.py index 9524edd..0389be5 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 -import os -import sys # Template by pypi-mobans import codecs import locale +import os import platform +import sys from shutil import rmtree -from setuptools import Command, setup, find_packages +from setuptools import Command, find_packages, setup PY2 = sys.version_info[0] == 2 PY26 = PY2 and sys.version_info[1] < 7 diff --git a/tests/requirements.txt b/tests/requirements.txt index f5f7679..787faf3 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,9 +1,11 @@ -{% extends 'tests/requirements.txt.jj2' %} -{%block extras %} +nose +mock;python_version<"3" +codecov +coverage +flake8 SQLAlchemy pyexcel>=0.2.0 pyexcel-xls>=0.1.0 moban black;python_version>="3.6" isort;python_version>="3.6" -{%endblock%} From 6435179a3ec724dd8b0fb755283301022b9573bc Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 10 Nov 2019 11:26:23 +0000 Subject: [PATCH 047/143] :sparkles: use inherited mobanfile. https://github.com/moremoban/moban/pull/348 --- .moban.yml | 17 +---------------- test.sh | 1 + 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.moban.yml b/.moban.yml index 5dd47b3..22a8079 100644 --- a/.moban.yml +++ b/.moban.yml @@ -1,25 +1,10 @@ +overrides: "git://github.com/pyexcel/pyexcel-mobans!/mobanfile.yaml" configuration: - configuration_dir: "git://github.com/pyexcel/pyexcel-mobans!/config" - template_dir: - - "git://github.com/moremoban/pypi-mobans.git?branch=dev&submodule=true!/templates" - - "git://github.com/pyexcel/pyexcel-mobans.git!/templates" - - "git://github.com/pyexcel/pyexcel-mobans.git!/statics" - - ".moban.d" configuration: pyexcel-io.yml targets: - "docs/source/conf.py": "docs/conf.py_t" - setup.py: io_setup.py.jj2 - .travis.yml: travis.yml.jj2 - - requirements.txt: requirements.txt.jj2 - "tests/requirements.txt": "tests/custom-requirements.txt.jj2" - - LICENSE: NEW_BSD_LICENSE.jj2 - - test.sh: test.script.jj2 - - test.bat: test.script.jj2 - README.rst: io_readme.rst.jj2 - "docs/source/index.rst": "docs/source/index.rst" - - output: CHANGELOG.rst - configuration: changelog.yml - template: CHANGELOG.rst.jj2 - - lint.sh: lint.script.jj2 - - ".github/FUNDING.yml": "FUNDING.yml" - - ".github/PULL_REQUEST_TEMPLATE.md": PULL_REQUEST_TEMPLATE.md diff --git a/test.sh b/test.sh index 9e24a8c..7526ea1 100644 --- a/test.sh +++ b/test.sh @@ -1,2 +1,3 @@ +#/bin/bash pip freeze nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io From e1351271c92b4dc185e3930024aa9ed8eb7f51c5 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 2 May 2020 23:38:49 +0100 Subject: [PATCH 048/143] :books: update project meta data --- .github/PULL_REQUEST_TEMPLATE.md | 2 ++ .travis.yml | 18 +++--------------- CHANGELOG.rst | 7 +++++++ README.rst | 21 +++++++++++++++------ changelog.yml | 6 ++++++ setup.py | 19 +++++++++---------- 6 files changed, 42 insertions(+), 31 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7b632ce..d5a2c03 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,6 +2,8 @@ With your PR, here is a check list: - [ ] Has Test cases written - [ ] Has all code lines tested +- [ ] Has `make format` been run? +- [ ] Has `moban` been run? - [ ] Passes all Travis CI builds - [ ] Has fair amount of documentation if your change is complex - [ ] run 'make format' so as to confirm the pyexcel organisation's coding style diff --git a/.travis.yml b/.travis.yml index cd67307..7e0336b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ notifications: python: - &pypy2 pypy2.7-6.0 - &pypy3 pypy3.5-6.0 - - 3.8-dev + - 3.8 - 3.7 - 3.6 - 3.5 @@ -16,25 +16,13 @@ stages: - lint - test -.disable_global: &disable_global - addons: false - cache: false - env: {} - python: false - before_install: false - install: false - before_script: false - script: false - after_success: false - after_failure: false - before_deploy: false - deploy: false .lint: &lint - <<: *disable_global git: submodules: false python: 3.6 + env: + - MINREQ=0 stage: lint script: make lint diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dbb6d48..9f22ef2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ Change log ================================================================================ +0.6.0 - tbd +-------------------------------------------------------------------------------- + +**removed** + +#. python 3.6 lower versions are no longer supported + 0.5.20 - 17.7.2019 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 0cc0c2f..aa566fa 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ Support the project ================================================================================ If your company has embedded pyexcel and its components into a revenue generating -product, please support me on `github `_, `patreon `_ +product, please support me on `patreon `_ or `bounty source `_ to maintain the project and develop it further. @@ -200,18 +200,27 @@ On Windows systems, please issue this command:: > test.bat -How to update test environment and update documentation + +Before you commit +------------------------------ + +Please run:: + + $ make format + +so as to beautify your code otherwise travis-ci may fail your unit test. + + +And make sure you would have run moban command --------------------------------------------------------- Additional steps are required: #. pip install moban #. make your changes in `.moban.d` directory, then issue command `moban` +#. moban -What is pyexcel-commons ---------------------------------- - -Many information that are shared across pyexcel projects, such as: this developer guide, license info, etc. are stored in `pyexcel-commons` project. +otherwise travis-ci may also fail your unit test. What is .moban.d --------------------------------- diff --git a/changelog.yml b/changelog.yml index 8dc5b05..395bb84 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: removed + details: + - 'python 3.6 lower versions are no longer supported' + version: 0.6.0 + date: tbd - changes: - action: updated details: diff --git a/setup.py b/setup.py index 0389be5..0a1e809 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -# Template by pypi-mobans +""" +Template by pypi-mobans +""" + +import os +import sys import codecs import locale -import os import platform -import sys from shutil import rmtree -from setuptools import Command, find_packages, setup +from setuptools import Command, setup, find_packages PY2 = sys.version_info[0] == 2 PY26 = PY2 and sys.version_info[1] < 7 @@ -60,24 +63,20 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", 'License :: OSI Approved :: BSD License', 'Programming Language :: Python :: Implementation :: PyPy' ] + INSTALL_REQUIRES = [ "lml>=0.0.4", ] SETUP_COMMANDS = {} -if PY26: - INSTALL_REQUIRES.append('ordereddict') - -PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests"]) +PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests", "tests.*"]) EXTRAS_REQUIRE = { "xls": ['pyexcel-xls>=0.5.0'], "xlsx": ['pyexcel-xlsx>=0.5.0'], From 61dd70ef9f8f57c12728d4676ff46a261806823f Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 2 May 2020 23:59:13 +0100 Subject: [PATCH 049/143] :green_heart: use latest dev branch of pyexcel for testing --- pyexcel-io.yml | 2 ++ rnd_requirements.txt | 2 +- setup.py | 12 +++++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 1545a79..180d0f1 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -27,3 +27,5 @@ keywords: - django - sqlalchemy description: A python library to read and write structured data in csv, zipped csv format and to/from databases +python_requires: ">=3.6" +min_python_version: "3.6" diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 676fcd7..37d8695 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,5 +1,5 @@ . https://github.com/chfw/lml/archive/master.zip -https://github.com/pyexcel/pyexcel/archive/master.zip +https://github.com/pyexcel/pyexcel/archive/dev.zip https://github.com/pyexcel/pyexcel-xls/archive/dev.zip diff --git a/setup.py b/setup.py index 0a1e809..b955a71 100644 --- a/setup.py +++ b/setup.py @@ -57,11 +57,11 @@ CLASSIFIERS = [ "Topic :: Software Development :: Libraries", "Programming Language :: Python", "Intended Audience :: Developers", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", + + "Programming Language :: Python :: 3 :: Only", + + + "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", @@ -70,6 +70,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: Implementation :: PyPy' ] +PYTHON_REQUIRES = ">=3.6" INSTALL_REQUIRES = [ "lml>=0.0.4", @@ -205,6 +206,7 @@ if __name__ == "__main__": long_description=read_files(*FILES), license=LICENSE, keywords=KEYWORDS, + python_requires=PYTHON_REQUIRES, extras_require=EXTRAS_REQUIRE, tests_require=["nose"], install_requires=INSTALL_REQUIRES, From feaefd540c5da2b429b3774c010501e9475a325d Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 3 May 2020 00:09:40 +0100 Subject: [PATCH 050/143] :fire: remove python 2 and pypy tests --- .moban.d/custom_travis.yml.jj2 | 15 +++++++++++++++ .moban.yml | 2 +- .travis.yml | 8 ++++---- 3 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 .moban.d/custom_travis.yml.jj2 diff --git a/.moban.d/custom_travis.yml.jj2 b/.moban.d/custom_travis.yml.jj2 new file mode 100644 index 0000000..73b434a --- /dev/null +++ b/.moban.d/custom_travis.yml.jj2 @@ -0,0 +1,15 @@ +{% extends "travis.yml.jj2" %} +{%block extra_matrix %} +matrix: + include: + - python: 3.6 + env: MINREQ=1 +{%endblock%} +{%block custom_python_versions%} +python: + - 3.8 + - 3.7 + - 3.6 +{%endblock%} +{%block pypi_deployment%} +{%endblock %} \ No newline at end of file diff --git a/.moban.yml b/.moban.yml index 22a8079..c4850d3 100644 --- a/.moban.yml +++ b/.moban.yml @@ -4,7 +4,7 @@ configuration: targets: - "docs/source/conf.py": "docs/conf.py_t" - setup.py: io_setup.py.jj2 - - .travis.yml: travis.yml.jj2 + - .travis.yml: custom_travis.yml.jj2 - "tests/requirements.txt": "tests/custom-requirements.txt.jj2" - README.rst: io_readme.rst.jj2 - "docs/source/index.rst": "docs/source/index.rst" diff --git a/.travis.yml b/.travis.yml index 7e0336b..1012eba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,13 +4,13 @@ language: python notifications: email: false python: - - &pypy2 pypy2.7-6.0 - - &pypy3 pypy3.5-6.0 - 3.8 - 3.7 - 3.6 - - 3.5 - - 2.7 +matrix: + include: + - python: 3.6 + env: MINREQ=1 stages: - lint From 276a2c1d02775551b6e583d7c3e0337e2666debf Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 3 May 2020 21:25:33 +0100 Subject: [PATCH 051/143] :fire: remove python 2 support . and now the cluttered code looks slightly better --- .../source/{conf.py => custom_conf.py.jj2} | 0 .moban.yml | 2 +- docs/source/conf.py | 43 +++++--- pyexcel_io/readers/csvr.py | 99 ++++++------------- 4 files changed, 61 insertions(+), 83 deletions(-) rename .moban.d/docs/source/{conf.py => custom_conf.py.jj2} (100%) diff --git a/.moban.d/docs/source/conf.py b/.moban.d/docs/source/custom_conf.py.jj2 similarity index 100% rename from .moban.d/docs/source/conf.py rename to .moban.d/docs/source/custom_conf.py.jj2 diff --git a/.moban.yml b/.moban.yml index c4850d3..44d697e 100644 --- a/.moban.yml +++ b/.moban.yml @@ -2,7 +2,7 @@ overrides: "git://github.com/pyexcel/pyexcel-mobans!/mobanfile.yaml" configuration: configuration: pyexcel-io.yml targets: - - "docs/source/conf.py": "docs/conf.py_t" + - "docs/source/conf.py": "docs/source/custom_conf.py.jj2" - setup.py: io_setup.py.jj2 - .travis.yml: custom_travis.yml.jj2 - "tests/requirements.txt": "tests/custom-requirements.txt.jj2" diff --git a/docs/source/conf.py b/docs/source/conf.py index 2c75e9f..29f8c7d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +DESCRIPTION = ( + 'A python library to read and write structured data in csv, zipped csv ' + + 'format and to/from databases' + + '' +) # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full @@ -17,7 +23,7 @@ # -- Project information ----------------------------------------------------- project = 'pyexcel-io' -copyright = 'copyright 2015-2019 Onni Software Ltd.' +copyright = '2015-2019 Onni Software Ltd.' author = 'C.W.' # The short X.Y version version = '0.6.0' @@ -32,20 +38,14 @@ release = '0.5.20' extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode',] # Add any paths that contain templates here, relative to this directory. -templates_path = ['templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '' +templates_path = ['_templates'] # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = 'Python' +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -63,10 +63,31 @@ html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['static'] +html_static_path = ['_static'] # -- Extension configuration ------------------------------------------------- # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/3/': None} \ No newline at end of file +intersphinx_mapping = {'https://docs.python.org/3/': None} +# TODO: html_theme not configurable upstream +html_theme = 'default' + + +def setup(app): + app.add_stylesheet('theme_overrides.css') + + + +# TODO: DESCRIPTION not configurable upstream +texinfo_documents = [ + ('index', 'pyexcel-io', + 'pyexcel-io Documentation', + 'Onni Software Ltd.', 'pyexcel-io', + DESCRIPTION, + 'Miscellaneous'), +] +intersphinx_mapping.update({ + 'pyexcel': ('http://pyexcel.readthedocs.io/en/latest/', None), +}) +master_doc = "index" diff --git a/pyexcel_io/readers/csvr.py b/pyexcel_io/readers/csvr.py index 50eedef..4c4d13e 100644 --- a/pyexcel_io/readers/csvr.py +++ b/pyexcel_io/readers/csvr.py @@ -11,7 +11,6 @@ import os import re import csv import glob -import codecs import pyexcel_io.service as service import pyexcel_io._compact as compact @@ -91,30 +90,6 @@ class CSVMemoryMapIterator(compact.Iterator): if line == "": raise StopIteration - if compact.PY2: - # python 2 requires utf-8 encoded string for reading - line = line.encode("utf-8") - return line - - -class UTF8Recorder(compact.Iterator): - """ - Iterator that reads an encoded stream and reencodes the input to UTF-8. - """ - - def __init__(self, file_handle, encoding): - self.__file_handle = file_handle - self.reader = codecs.getreader(encoding)(file_handle) - - def close(self): - self.__file_handle.close() - - def __iter__(self): - return self - - def __next__(self): - # python 2 requires utf-8 encoded string for reading - line = next(self.reader).encode("utf-8") return line @@ -196,14 +171,9 @@ class CSVFileReader(CSVSheetReader): """ read csv from phyical file """ def get_file_handle(self): - unicode_reader = None - if compact.PY2: - file_handle = open(self._native_sheet.payload, "rb") - unicode_reader = UTF8Recorder(file_handle, self._encoding) - else: - unicode_reader = open( - self._native_sheet.payload, "r", encoding=self._encoding - ) + unicode_reader = open( + self._native_sheet.payload, "r", encoding=self._encoding + ) return unicode_reader @@ -211,26 +181,17 @@ class CSVinMemoryReader(CSVSheetReader): """ read csv file from memory """ def get_file_handle(self): - unicode_reader = None - if compact.PY2: - if hasattr(self._native_sheet.payload, "read"): - unicode_reader = UTF8Recorder( - self._native_sheet.payload, self._encoding - ) - else: - unicode_reader = self._native_sheet.payload + if isinstance(self._native_sheet.payload, compact.BytesIO): + # please note that + # if the end developer feed us bytesio in python3 + # we will do the conversion to StriongIO but that + # comes at a cost. + content = self._native_sheet.payload.read() + unicode_reader = compact.StringIO( + content.decode(self._encoding) + ) else: - if isinstance(self._native_sheet.payload, compact.BytesIO): - # please note that - # if the end developer feed us bytesio in python3 - # we will do the conversion to StriongIO but that - # comes at a cost. - content = self._native_sheet.payload.read() - unicode_reader = compact.StringIO( - content.decode(self._encoding) - ) - else: - unicode_reader = self._native_sheet.payload + unicode_reader = self._native_sheet.payload return unicode_reader @@ -259,26 +220,22 @@ class CSVBookReader(BookReader): self._native_book = self._load_from_stream() def open_content(self, file_content, **keywords): - try: - import mmap + import mmap - encoding = keywords.get("encoding", "utf-8") - if isinstance(file_content, mmap.mmap): - # load from mmap - self.__multiple_sheets = keywords.get("multiple_sheets", False) - self._file_stream = CSVMemoryMapIterator( - file_content, encoding - ) - self._keywords = keywords - self._native_book = self._load_from_stream() - else: - if compact.PY3_ABOVE: - if isinstance(file_content, bytes): - file_content = file_content.decode(encoding) - # else python 2.7 does not care about bytes nor str - BookReader.open_content(self, file_content, **keywords) - except ImportError: - # python 2.6 or Google app engine + encoding = keywords.get("encoding", "utf-8") + if isinstance(file_content, mmap.mmap): + # load from mmap + self.__multiple_sheets = keywords.get("multiple_sheets", False) + self._file_stream = CSVMemoryMapIterator( + file_content, encoding + ) + self._keywords = keywords + self._native_book = self._load_from_stream() + else: + if compact.PY3_ABOVE: + if isinstance(file_content, bytes): + file_content = file_content.decode(encoding) + # else python 2.7 does not care about bytes nor str BookReader.open_content(self, file_content, **keywords) def read_sheet(self, native_sheet): From fc38e2bb14721751d512e710b998552a712f6a22 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 4 May 2020 20:32:37 +0100 Subject: [PATCH 052/143] :hammer: code refactoring --- pyexcel_io/book.py | 1 - pyexcel_io/plugins.py | 5 +++-- pyexcel_io/readers/csvr.py | 15 +++++-------- pyexcel_io/readers/csvz.py | 1 - pyexcel_io/readers/tsv.py | 1 - pyexcel_io/readers/tsvz.py | 1 - pyexcel_io/writers/csvw.py | 24 ++++++--------------- pyexcel_io/writers/csvz.py | 1 - pyexcel_io/writers/tsv.py | 1 - pyexcel_io/writers/tsvz.py | 1 - tests/test_base.py | 3 +-- tests/test_book.py | 5 ++--- tests/test_csv_book.py | 3 +-- tests/test_django_book.py | 3 +-- tests/test_filter.py | 3 +-- tests/test_io.py | 3 +-- tests/test_issues.py | 4 ++-- tests/test_new_csv_book.py | 3 +-- tests/test_new_csvz_book.py | 3 +-- tests/test_pyexcel_integration.py | 1 + tests/test_renderer.py | 3 +-- tests/test_service.py | 5 ++--- tests/test_sheet.py | 3 +-- tests/test_sql_book.py | 35 +++++++++++++++---------------- 24 files changed, 47 insertions(+), 81 deletions(-) diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index d728666..40eaa53 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -9,7 +9,6 @@ """ import pyexcel_io.manager as manager from pyexcel_io._compact import PY2, OrderedDict, isstream - from .constants import MESSAGE_ERROR_03, MESSAGE_WRONG_IO_INSTANCE diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 2c350ef..2cce916 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -7,12 +7,13 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +from lml.loader import scan_plugins_regex +from lml.plugin import PluginInfo, PluginManager, PluginInfoChain + import pyexcel_io.utils as ioutils import pyexcel_io.manager as manager import pyexcel_io.constants as constants import pyexcel_io.exceptions as exceptions -from lml.loader import scan_plugins_regex -from lml.plugin import PluginInfo, PluginManager, PluginInfoChain ERROR_MESSAGE_FORMATTER = "one of these plugins for %s data in '%s': %s" UPGRADE_MESSAGE = "Please upgrade the plugin '%s' according to \ diff --git a/pyexcel_io/readers/csvr.py b/pyexcel_io/readers/csvr.py index 4c4d13e..e2cc20a 100644 --- a/pyexcel_io/readers/csvr.py +++ b/pyexcel_io/readers/csvr.py @@ -187,9 +187,7 @@ class CSVinMemoryReader(CSVSheetReader): # we will do the conversion to StriongIO but that # comes at a cost. content = self._native_sheet.payload.read() - unicode_reader = compact.StringIO( - content.decode(self._encoding) - ) + unicode_reader = compact.StringIO(content.decode(self._encoding)) else: unicode_reader = self._native_sheet.payload @@ -226,16 +224,13 @@ class CSVBookReader(BookReader): if isinstance(file_content, mmap.mmap): # load from mmap self.__multiple_sheets = keywords.get("multiple_sheets", False) - self._file_stream = CSVMemoryMapIterator( - file_content, encoding - ) + self._file_stream = CSVMemoryMapIterator(file_content, encoding) self._keywords = keywords self._native_book = self._load_from_stream() else: - if compact.PY3_ABOVE: - if isinstance(file_content, bytes): - file_content = file_content.decode(encoding) - # else python 2.7 does not care about bytes nor str + if isinstance(file_content, bytes): + file_content = file_content.decode(encoding) + BookReader.open_content(self, file_content, **keywords) def read_sheet(self, native_sheet): diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index c956eea..24410b1 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -12,7 +12,6 @@ import zipfile from pyexcel_io.book import BookReader from pyexcel_io._compact import PY2, StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ - from .csvr import NamedContent, CSVinMemoryReader diff --git a/pyexcel_io/readers/tsv.py b/pyexcel_io/readers/tsv.py index 35829ae..8ac135a 100644 --- a/pyexcel_io/readers/tsv.py +++ b/pyexcel_io/readers/tsv.py @@ -8,7 +8,6 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants - from .csvr import CSVBookReader diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py index 9dc7d6d..c7b7310 100644 --- a/pyexcel_io/readers/tsvz.py +++ b/pyexcel_io/readers/tsvz.py @@ -8,7 +8,6 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT - from .csvz import CSVZipBookReader diff --git a/pyexcel_io/writers/csvw.py b/pyexcel_io/writers/csvw.py index 910ecfd..18d3e71 100644 --- a/pyexcel_io/writers/csvw.py +++ b/pyexcel_io/writers/csvw.py @@ -108,16 +108,10 @@ class CSVFileWriter(CSVSheetWriter): ) else: file_name = self._native_book - if compact.PY2: - self.file_handle = open(file_name, "wb") - self.writer = UnicodeWriter( - self.file_handle, encoding=self._encoding, **self._keywords - ) - else: - self.file_handle = open( - file_name, "w", newline="", encoding=self._encoding - ) - self.writer = csv.writer(self.file_handle, **self._keywords) + self.file_handle = open( + file_name, "w", newline="", encoding=self._encoding + ) + self.writer = csv.writer(self.file_handle, **self._keywords) class CSVMemoryWriter(CSVSheetWriter): @@ -143,14 +137,8 @@ class CSVMemoryWriter(CSVSheetWriter): ) def set_sheet_name(self, name): - if compact.PY2: - self.file_handle = self._native_book - self.writer = UnicodeWriter( - self.file_handle, encoding=self._encoding, **self._keywords - ) - else: - self.file_handle = self._native_book - self.writer = csv.writer(self.file_handle, **self._keywords) + self.file_handle = self._native_book + self.writer = csv.writer(self.file_handle, **self._keywords) if not self._single_sheet_in_book: self.writer.writerow( [ diff --git a/pyexcel_io/writers/csvz.py b/pyexcel_io/writers/csvz.py index 3c3c3de..76d291b 100644 --- a/pyexcel_io/writers/csvz.py +++ b/pyexcel_io/writers/csvz.py @@ -12,7 +12,6 @@ import zipfile from pyexcel_io.book import BookWriter from pyexcel_io._compact import PY2, StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ, DEFAULT_SHEET_NAME - from .csvw import UnicodeWriter, CSVSheetWriter diff --git a/pyexcel_io/writers/tsv.py b/pyexcel_io/writers/tsv.py index 757cad2..778372b 100644 --- a/pyexcel_io/writers/tsv.py +++ b/pyexcel_io/writers/tsv.py @@ -8,7 +8,6 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants - from .csvw import CSVBookWriter diff --git a/pyexcel_io/writers/tsvz.py b/pyexcel_io/writers/tsvz.py index b12814d..255730a 100644 --- a/pyexcel_io/writers/tsvz.py +++ b/pyexcel_io/writers/tsvz.py @@ -8,7 +8,6 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT - from .csvz import CSVZipBookWriter diff --git a/tests/test_base.py b/tests/test_base.py index df4dfb1..a7c890e 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,9 +1,8 @@ +from nose.tools import raises from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetReader, SheetWriter, NamedContent from pyexcel_io.utils import is_empty_array -from nose.tools import raises - @raises(NotImplementedError) def test_book_writer(): diff --git a/tests/test_book.py b/tests/test_book.py index 14755a2..375eadc 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -1,3 +1,5 @@ +from nose import SkipTest +from nose.tools import raises from pyexcel_io.book import ( BookReader, BookWriter, @@ -6,9 +8,6 @@ from pyexcel_io.book import ( ) from pyexcel_io._compact import PY2, BytesIO, StringIO -from nose import SkipTest -from nose.tools import raises - @raises(NotImplementedError) def test_rwinterface(): diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index ecbacb6..c530f7b 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -6,6 +6,7 @@ from textwrap import dedent from unittest import TestCase import pyexcel_io.manager as manager +from nose.tools import eq_, raises from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import PY2, BytesIO, StringIO from pyexcel_io.readers.csvr import ( @@ -15,8 +16,6 @@ from pyexcel_io.readers.csvr import ( ) from pyexcel_io.writers.csvw import CSVFileWriter, CSVMemoryWriter -from nose.tools import eq_, raises - class TestReaders(TestCase): def setUp(self): diff --git a/tests/test_django_book.py b/tests/test_django_book.py index 7ad4a6a..95239df 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -1,3 +1,4 @@ +from nose.tools import eq_, raises from pyexcel_io import save_data from pyexcel_io._compact import OrderedDict from pyexcel_io.constants import DB_DJANGO @@ -16,8 +17,6 @@ from pyexcel_io.database.importers.django import ( DjangoModelWriter, ) -from nose.tools import eq_, raises - class Package: def __init__(self, raiseException=False, **keywords): diff --git a/tests/test_filter.py b/tests/test_filter.py index a3d3591..3af8eac 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -1,11 +1,10 @@ import os import pyexcel_io.constants as constants +from nose.tools import eq_ from pyexcel_io import get_data, save_data from pyexcel_io.utils import _index_filter -from nose.tools import eq_ - def test_index_filter(): current_index, start, limit, expected = (0, 1, -1, constants.SKIP_DATA) diff --git a/tests/test_io.py b/tests/test_io.py index 1870c74..0142fcb 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -6,12 +6,11 @@ from unittest import TestCase import pyexcel_io.manager as manager import pyexcel_io.exceptions as exceptions +from nose.tools import eq_, raises from pyexcel_io import get_data, iget_data, save_data from pyexcel_io.io import load_data, get_writer from pyexcel_io._compact import BytesIO, StringIO, OrderedDict, is_string -from nose.tools import eq_, raises - PY2 = sys.version_info[0] == 2 diff --git a/tests/test_issues.py b/tests/test_issues.py index 09582e0..7a436d2 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -4,11 +4,11 @@ import os import pyexcel as p -from pyexcel_io import get_data, save_data -from pyexcel_io._compact import PY26 from nose import SkipTest from nose.tools import eq_ +from pyexcel_io import get_data, save_data +from pyexcel_io._compact import PY26 IN_TRAVIS = "TRAVIS" in os.environ diff --git a/tests/test_new_csv_book.py b/tests/test_new_csv_book.py index 930a58a..857f4ee 100644 --- a/tests/test_new_csv_book.py +++ b/tests/test_new_csv_book.py @@ -3,14 +3,13 @@ from textwrap import dedent from unittest import TestCase import pyexcel_io.manager as manager +from nose.tools import raises from pyexcel_io._compact import OrderedDict from pyexcel_io.readers.tsv import TSVBookReader from pyexcel_io.writers.tsv import TSVBookWriter from pyexcel_io.readers.csvr import CSVBookReader from pyexcel_io.writers.csvw import CSVBookWriter -from nose.tools import raises - class TestCSVReaders(TestCase): file_type = "csv" diff --git a/tests/test_new_csvz_book.py b/tests/test_new_csvz_book.py index b73faed..fdf8620 100644 --- a/tests/test_new_csvz_book.py +++ b/tests/test_new_csvz_book.py @@ -5,6 +5,7 @@ import zipfile from unittest import TestCase import pyexcel_io.manager as manager +from nose.tools import raises from pyexcel_io import save_data from pyexcel_io._compact import OrderedDict from pyexcel_io.readers.csvz import CSVZipBookReader @@ -12,8 +13,6 @@ from pyexcel_io.readers.tsvz import TSVZipBookReader from pyexcel_io.writers.csvz import CSVZipBookWriter from pyexcel_io.writers.tsvz import TSVZipBookWriter -from nose.tools import raises - PY2 = sys.version_info[0] == 2 diff --git a/tests/test_pyexcel_integration.py b/tests/test_pyexcel_integration.py index 963da21..e198c74 100644 --- a/tests/test_pyexcel_integration.py +++ b/tests/test_pyexcel_integration.py @@ -4,6 +4,7 @@ from textwrap import dedent from unittest import TestCase import pyexcel as pe + from pyexcel_io._compact import text_type diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 209bad9..5be2174 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -1,8 +1,7 @@ import os -from pyexcel_io import get_data, save_data - from nose.tools import eq_ +from pyexcel_io import get_data, save_data class TestRenderer: diff --git a/tests/test_service.py b/tests/test_service.py index 5971708..cb6a7bd 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,3 +1,5 @@ +from nose import SkipTest +from nose.tools import eq_, raises from pyexcel_io.service import ( ODS_WRITE_FORMAT_COVERSION, date_value, @@ -10,9 +12,6 @@ from pyexcel_io.service import ( from pyexcel_io._compact import PY2 from pyexcel_io.exceptions import IntegerAccuracyLossError -from nose import SkipTest -from nose.tools import eq_, raises - def test_date_util_parse(): value = "2015-08-17T19:20:00" diff --git a/tests/test_sheet.py b/tests/test_sheet.py index d82cb30..789317a 100644 --- a/tests/test_sheet.py +++ b/tests/test_sheet.py @@ -1,7 +1,6 @@ import pyexcel_io.constants as constants -from pyexcel_io.sheet import SheetReader, SheetWriter - from nose.tools import eq_ +from pyexcel_io.sheet import SheetReader, SheetWriter class MyWriter(SheetWriter): diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 47bc5c9..6927056 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -3,24 +3,6 @@ import json import datetime import platform -from pyexcel_io._compact import OrderedDict -from pyexcel_io.database.common import ( - SQLTableExporter, - SQLTableImporter, - SQLTableExportAdapter, - SQLTableImportAdapter, -) -from pyexcel_io.database.querysets import QuerysetsReader -from pyexcel_io.database.exporters.sqlalchemy import ( - SQLBookReader, - SQLTableReader, -) -from pyexcel_io.database.importers.sqlalchemy import ( - SQLBookWriter, - SQLTableWriter, - PyexcelSQLSkipRowException, -) - from nose.tools import eq_, raises from sqlalchemy import ( Date, @@ -33,7 +15,24 @@ from sqlalchemy import ( create_engine, ) from sqlalchemy.orm import backref, relationship, sessionmaker +from pyexcel_io._compact import OrderedDict +from pyexcel_io.database.common import ( + SQLTableExporter, + SQLTableImporter, + SQLTableExportAdapter, + SQLTableImportAdapter, +) from sqlalchemy.ext.declarative import declarative_base +from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.database.exporters.sqlalchemy import ( + SQLBookReader, + SQLTableReader, +) +from pyexcel_io.database.importers.sqlalchemy import ( + SQLBookWriter, + SQLTableWriter, + PyexcelSQLSkipRowException, +) PY3 = sys.version_info[0] == 3 PY36 = PY3 and sys.version_info[1] == 6 From 9f3e1bfc007f0645e8a665ac57f571f30523306d Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 4 May 2020 20:34:45 +0100 Subject: [PATCH 053/143] :fire: remove python 2 code --- pyexcel_io/writers/csvw.py | 35 ----------------------------------- pyexcel_io/writers/csvz.py | 13 ++++--------- 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/pyexcel_io/writers/csvw.py b/pyexcel_io/writers/csvw.py index 18d3e71..ff4ead1 100644 --- a/pyexcel_io/writers/csvw.py +++ b/pyexcel_io/writers/csvw.py @@ -8,7 +8,6 @@ :license: New BSD License, see LICENSE for more details """ import csv -import codecs import pyexcel_io._compact as compact import pyexcel_io.constants as constants @@ -16,40 +15,6 @@ from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetWriter -class UnicodeWriter(object): - """ - A CSV writer which will write rows to CSV file "f", - which is encoded in the given encoding. - """ - - def __init__(self, file_handle, encoding="utf-8", **kwds): - # Redirect output to a queue - self.queue = compact.StringIO() - self.writer = csv.writer(self.queue, **kwds) - self.stream = file_handle - self.encoder = codecs.getincrementalencoder(encoding)() - - def writerow(self, row): - """ write row into the csv file """ - self.writer.writerow( - [compact.text_type(s).encode("utf-8") for s in row] - ) - # Fetch UTF-8 output from the queue ... - data = self.queue.getvalue() - data = data.decode("utf-8") - # ... and reencode it into the target encoding - data = self.encoder.encode(data) - # write to the target stream - self.stream.write(data) - # empty queue - self.queue.truncate(0) - - def writerows(self, rows): - """ write multiple rows into csv file """ - for row in rows: - self.writerow(row) - - class CSVSheetWriter(SheetWriter): """ csv file writer diff --git a/pyexcel_io/writers/csvz.py b/pyexcel_io/writers/csvz.py index 76d291b..3012577 100644 --- a/pyexcel_io/writers/csvz.py +++ b/pyexcel_io/writers/csvz.py @@ -10,9 +10,9 @@ import zipfile from pyexcel_io.book import BookWriter -from pyexcel_io._compact import PY2, StringIO +from pyexcel_io._compact import StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ, DEFAULT_SHEET_NAME -from .csvw import UnicodeWriter, CSVSheetWriter +from .csvw import CSVSheetWriter class CSVZipSheetWriter(CSVSheetWriter): @@ -25,14 +25,9 @@ class CSVZipSheetWriter(CSVSheetWriter): def set_sheet_name(self, name): self.content = StringIO() - if PY2: - self.writer = UnicodeWriter( - self.content, encoding=self._encoding, **self._keywords - ) - else: - import csv + import csv - self.writer = csv.writer(self.content, **self._keywords) + self.writer = csv.writer(self.content, **self._keywords) def close(self): file_name = "%s.%s" % (self._native_sheet, self.file_extension) From 44a8615e1c8dc2887a4648bf74b6b3880dbad5d1 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 4 May 2020 20:37:21 +0100 Subject: [PATCH 054/143] :fire: remove python 2 support --- pyexcel_io/io.py | 7 ++----- pyexcel_io/service.py | 5 ----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 20f5f1a..c6d84c4 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -13,7 +13,7 @@ from types import GeneratorType from pyexcel_io import constants from pyexcel_io.plugins import READERS, WRITERS -from pyexcel_io._compact import PY2, isstream +from pyexcel_io._compact import isstream from pyexcel_io.exceptions import NoSupportingPluginFound @@ -113,10 +113,7 @@ def save_data(afile, data, file_type=None, **keywords): single_sheet_in_book = True to_store = {constants.DEFAULT_SHEET_NAME: data} else: - if PY2: - keys = data.keys() - else: - keys = list(data.keys()) + keys = list(data.keys()) single_sheet_in_book = len(keys) == 1 no_file_type = isstream(afile) and file_type is None diff --git a/pyexcel_io/service.py b/pyexcel_io/service.py index a074c27..5973ae3 100644 --- a/pyexcel_io/service.py +++ b/pyexcel_io/service.py @@ -12,7 +12,6 @@ import math import datetime from pyexcel_io import constants, exceptions -from pyexcel_io._compact import PY2 def has_no_digits_in_float(value): @@ -177,10 +176,6 @@ ODS_WRITE_FORMAT_COVERSION = { bool: "boolean", } -if PY2: - ODS_WRITE_FORMAT_COVERSION[unicode] = "string" # noqa: F821 - ODS_WRITE_FORMAT_COVERSION[long] = "long" # noqa: F821 - VALUE_CONVERTERS = { "float": float_value, From f1b715112b3dd06c39120e1e09392542c2cdd61f Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 4 May 2020 20:40:19 +0100 Subject: [PATCH 055/143] :fire: remove python 2 code --- pyexcel_io/book.py | 50 ++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index 40eaa53..24254ef 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -85,26 +85,15 @@ class BookReader(RWInterface): keywords are passed on to individual readers """ if isstream(file_stream): - if PY2: - if hasattr(file_stream, "seek"): - file_stream.seek(0) - else: - # python 2 - # Hei zipfile in odfpy would do a seek - # but stream from urlib cannot do seek - file_stream = _convert_content_to_stream( - file_stream.read(), self._file_type - ) - else: - from io import UnsupportedOperation + from io import UnsupportedOperation - try: - file_stream.seek(0) - except UnsupportedOperation: - # python 3 - file_stream = _convert_content_to_stream( - file_stream.read(), self._file_type - ) + try: + file_stream.seek(0) + except UnsupportedOperation: + # python 3 + file_stream = _convert_content_to_stream( + file_stream.read(), self._file_type + ) self._file_stream = file_stream self._keywords = keywords @@ -231,18 +220,17 @@ class BookWriter(RWInterface): def _convert_content_to_stream(file_content, file_type): stream = manager.get_io(file_type) - if not PY2: - target_content_type = manager.get_io_type(file_type) - needs_encode = target_content_type == "bytes" and not isinstance( - file_content, bytes - ) - needs_decode = target_content_type == "string" and isinstance( - file_content, bytes - ) - if needs_encode: - file_content = file_content.encode("utf-8") - elif needs_decode: - file_content = file_content.decode("utf-8") + target_content_type = manager.get_io_type(file_type) + needs_encode = target_content_type == "bytes" and not isinstance( + file_content, bytes + ) + needs_decode = target_content_type == "string" and isinstance( + file_content, bytes + ) + if needs_encode: + file_content = file_content.encode("utf-8") + elif needs_decode: + file_content = file_content.decode("utf-8") stream.write(file_content) stream.seek(0) return stream From 1440df833b220aa43e2dee6454dd89a616866189 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 4 May 2020 21:05:42 +0100 Subject: [PATCH 056/143] :hammer: new writer implementation prototype --- pyexcel_io/io.py | 5 +++-- pyexcel_io/writer.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 pyexcel_io/writer.py diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index c6d84c4..7dbefc3 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -12,7 +12,8 @@ import warnings from types import GeneratorType from pyexcel_io import constants -from pyexcel_io.plugins import READERS, WRITERS +from pyexcel_io.plugins import READERS +from pyexcel_io.writer import Writer from pyexcel_io._compact import isstream from pyexcel_io.exceptions import NoSupportingPluginFound @@ -236,7 +237,7 @@ def get_writer( file_type_given = False - writer = WRITERS.get_a_plugin(file_type, library) + writer = Writer(file_type, library) if file_name: if file_type_given: writer.open_content(file_name, **keywords) diff --git a/pyexcel_io/writer.py b/pyexcel_io/writer.py new file mode 100644 index 0000000..bdaf982 --- /dev/null +++ b/pyexcel_io/writer.py @@ -0,0 +1,30 @@ +from pyexcel_io.plugins import WRITERS + + +class Writer(object): + def __init__(self, file_type, library): + self.writer = WRITERS.get_a_plugin(file_type, library) + + def open(self, file_name, **keywords): + self.writer.open(file_name, **keywords) + + def open_stream(self, file_stream, **keywords): + self.writer.open_stream(file_stream, **keywords) + + def open_content(self, file_stream, **keywords): + self.writer.open_content(file_stream, **keywords) + + def write(self, incoming_dict): + self.writer.write(incoming_dict) + + def create_sheet(self, sheet_name): + return self.writer.create_sheet(sheet_name) + + def close(self): + self.writer.close() + + def __enter__(self): + return self + + def __exit__(self, a_type, value, traceback): + self.close() From 9640ed411964f6a7316b7795a09dc163db0cfbed Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 4 May 2020 21:13:12 +0100 Subject: [PATCH 057/143] :hammer: new reader implementation --- .github/workflows/pythonpublish.yml | 26 +++++++++++++++++++++ pyexcel_io/io.py | 4 ++-- pyexcel_io/reader.py | 36 +++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pythonpublish.yml create mode 100644 pyexcel_io/reader.py diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml new file mode 100644 index 0000000..9e7ec42 --- /dev/null +++ b/.github/workflows/pythonpublish.yml @@ -0,0 +1,26 @@ +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 7dbefc3..529ceea 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -12,7 +12,7 @@ import warnings from types import GeneratorType from pyexcel_io import constants -from pyexcel_io.plugins import READERS +from pyexcel_io.reader import Reader from pyexcel_io.writer import Writer from pyexcel_io._compact import isstream from pyexcel_io.exceptions import NoSupportingPluginFound @@ -169,7 +169,7 @@ def load_data( raise Exception(constants.MESSAGE_FILE_NAME_SHOULD_BE_STRING) try: - reader = READERS.get_a_plugin(file_type, library) + reader = Reader(file_type, library) except NoSupportingPluginFound: if file_name: if os.path.exists(file_name): diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py new file mode 100644 index 0000000..1161352 --- /dev/null +++ b/pyexcel_io/reader.py @@ -0,0 +1,36 @@ +from pyexcel_io.plugins import READERS + + +class Reader(object): + def __init__(self, file_type, library=None): + self.reader = READERS.get_a_plugin(file_type, library) + + def open(self, file_name, **keywords): + return self.reader.open(file_name, **keywords) + + def open_content(self, file_content, **keywords): + return self.reader.open_content(file_content, **keywords) + + def open_stream(self, file_stream, **keywords): + return self.reader.open_stream(file_stream, **keywords) + + def read_sheet_by_index(self, sheet_index): + return self.reader.read_sheet_by_index(sheet_index) + + def read_sheet_by_name(self, sheet_name): + return self.reader.read_sheet_by_name(sheet_name) + + def read_many(self, sheets): + return self.reader.read_many(sheets) + + def read_all(self): + return self.reader.read_all() + + def close(self): + return self.reader.close() + + def __enter__(self): + return self + + def __exit__(self, a_type, value, traceback): + self.close() From 725aa26ed6e5a6a993e1474810881906bd12ba1d Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 22:09:57 +0100 Subject: [PATCH 058/143] keep pyexcel-xx 0.5.0 compactibility --- README.rst | 5 ++++- pyexcel_io/book.py | 3 ++- pyexcel_io/io.py | 4 ++++ pyexcel_io/plugins.py | 5 ++--- pyexcel_io/readers/csvz.py | 1 + pyexcel_io/readers/tsv.py | 1 + pyexcel_io/readers/tsvz.py | 1 + pyexcel_io/writers/csvz.py | 1 + pyexcel_io/writers/tsv.py | 1 + pyexcel_io/writers/tsvz.py | 1 + tests/test_base.py | 3 ++- tests/test_book.py | 5 +++-- tests/test_csv_book.py | 3 ++- tests/test_django_book.py | 3 ++- tests/test_filter.py | 3 ++- tests/test_io.py | 3 ++- tests/test_issues.py | 4 ++-- tests/test_new_csv_book.py | 3 ++- tests/test_new_csvz_book.py | 3 ++- tests/test_pyexcel_integration.py | 1 - tests/test_renderer.py | 3 ++- tests/test_service.py | 5 +++-- tests/test_sheet.py | 3 ++- tests/test_sql_book.py | 35 ++++++++++++++++--------------- 24 files changed, 62 insertions(+), 38 deletions(-) diff --git a/README.rst b/README.rst index aa566fa..0db73a2 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,9 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://raw.githubusercontent.com/pyexcel/pyexcel.github.io/master/images/patreon.png :target: https://www.patreon.com/chfw +.. image:: https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg + :target: https://awesome-python.com/#specific-formats-processing + .. image:: https://travis-ci.org/pyexcel/pyexcel-io.svg?branch=master :target: http://travis-ci.org/pyexcel/pyexcel-io @@ -33,7 +36,7 @@ Support the project ================================================================================ If your company has embedded pyexcel and its components into a revenue generating -product, please support me on `patreon `_ +product, please support me on github, `patreon `_ or `bounty source `_ to maintain the project and develop it further. diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index 24254ef..46d9b87 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -8,7 +8,8 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.manager as manager -from pyexcel_io._compact import PY2, OrderedDict, isstream +from pyexcel_io._compact import OrderedDict, isstream + from .constants import MESSAGE_ERROR_03, MESSAGE_WRONG_IO_INSTANCE diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 529ceea..f20ed2c 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -247,3 +247,7 @@ def get_writer( writer.open_stream(file_stream, **keywords) # else: is resolved by earlier raise statement return writer + + +# backward compactibility +write_data = save_data diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 2cce916..2c350ef 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -7,13 +7,12 @@ :copyright: (c) 2014-2017 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from lml.loader import scan_plugins_regex -from lml.plugin import PluginInfo, PluginManager, PluginInfoChain - import pyexcel_io.utils as ioutils import pyexcel_io.manager as manager import pyexcel_io.constants as constants import pyexcel_io.exceptions as exceptions +from lml.loader import scan_plugins_regex +from lml.plugin import PluginInfo, PluginManager, PluginInfoChain ERROR_MESSAGE_FORMATTER = "one of these plugins for %s data in '%s': %s" UPGRADE_MESSAGE = "Please upgrade the plugin '%s' according to \ diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index 24410b1..c956eea 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -12,6 +12,7 @@ import zipfile from pyexcel_io.book import BookReader from pyexcel_io._compact import PY2, StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ + from .csvr import NamedContent, CSVinMemoryReader diff --git a/pyexcel_io/readers/tsv.py b/pyexcel_io/readers/tsv.py index 8ac135a..35829ae 100644 --- a/pyexcel_io/readers/tsv.py +++ b/pyexcel_io/readers/tsv.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants + from .csvr import CSVBookReader diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py index c7b7310..9dc7d6d 100644 --- a/pyexcel_io/readers/tsvz.py +++ b/pyexcel_io/readers/tsvz.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT + from .csvz import CSVZipBookReader diff --git a/pyexcel_io/writers/csvz.py b/pyexcel_io/writers/csvz.py index 3012577..381168f 100644 --- a/pyexcel_io/writers/csvz.py +++ b/pyexcel_io/writers/csvz.py @@ -12,6 +12,7 @@ import zipfile from pyexcel_io.book import BookWriter from pyexcel_io._compact import StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ, DEFAULT_SHEET_NAME + from .csvw import CSVSheetWriter diff --git a/pyexcel_io/writers/tsv.py b/pyexcel_io/writers/tsv.py index 778372b..757cad2 100644 --- a/pyexcel_io/writers/tsv.py +++ b/pyexcel_io/writers/tsv.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants + from .csvw import CSVBookWriter diff --git a/pyexcel_io/writers/tsvz.py b/pyexcel_io/writers/tsvz.py index 255730a..b12814d 100644 --- a/pyexcel_io/writers/tsvz.py +++ b/pyexcel_io/writers/tsvz.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT + from .csvz import CSVZipBookWriter diff --git a/tests/test_base.py b/tests/test_base.py index a7c890e..df4dfb1 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,8 +1,9 @@ -from nose.tools import raises from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetReader, SheetWriter, NamedContent from pyexcel_io.utils import is_empty_array +from nose.tools import raises + @raises(NotImplementedError) def test_book_writer(): diff --git a/tests/test_book.py b/tests/test_book.py index 375eadc..14755a2 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -1,5 +1,3 @@ -from nose import SkipTest -from nose.tools import raises from pyexcel_io.book import ( BookReader, BookWriter, @@ -8,6 +6,9 @@ from pyexcel_io.book import ( ) from pyexcel_io._compact import PY2, BytesIO, StringIO +from nose import SkipTest +from nose.tools import raises + @raises(NotImplementedError) def test_rwinterface(): diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index c530f7b..ecbacb6 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -6,7 +6,6 @@ from textwrap import dedent from unittest import TestCase import pyexcel_io.manager as manager -from nose.tools import eq_, raises from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import PY2, BytesIO, StringIO from pyexcel_io.readers.csvr import ( @@ -16,6 +15,8 @@ from pyexcel_io.readers.csvr import ( ) from pyexcel_io.writers.csvw import CSVFileWriter, CSVMemoryWriter +from nose.tools import eq_, raises + class TestReaders(TestCase): def setUp(self): diff --git a/tests/test_django_book.py b/tests/test_django_book.py index 95239df..7ad4a6a 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -1,4 +1,3 @@ -from nose.tools import eq_, raises from pyexcel_io import save_data from pyexcel_io._compact import OrderedDict from pyexcel_io.constants import DB_DJANGO @@ -17,6 +16,8 @@ from pyexcel_io.database.importers.django import ( DjangoModelWriter, ) +from nose.tools import eq_, raises + class Package: def __init__(self, raiseException=False, **keywords): diff --git a/tests/test_filter.py b/tests/test_filter.py index 3af8eac..a3d3591 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -1,10 +1,11 @@ import os import pyexcel_io.constants as constants -from nose.tools import eq_ from pyexcel_io import get_data, save_data from pyexcel_io.utils import _index_filter +from nose.tools import eq_ + def test_index_filter(): current_index, start, limit, expected = (0, 1, -1, constants.SKIP_DATA) diff --git a/tests/test_io.py b/tests/test_io.py index 0142fcb..1870c74 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -6,11 +6,12 @@ from unittest import TestCase import pyexcel_io.manager as manager import pyexcel_io.exceptions as exceptions -from nose.tools import eq_, raises from pyexcel_io import get_data, iget_data, save_data from pyexcel_io.io import load_data, get_writer from pyexcel_io._compact import BytesIO, StringIO, OrderedDict, is_string +from nose.tools import eq_, raises + PY2 = sys.version_info[0] == 2 diff --git a/tests/test_issues.py b/tests/test_issues.py index 7a436d2..09582e0 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -4,11 +4,11 @@ import os import pyexcel as p +from pyexcel_io import get_data, save_data +from pyexcel_io._compact import PY26 from nose import SkipTest from nose.tools import eq_ -from pyexcel_io import get_data, save_data -from pyexcel_io._compact import PY26 IN_TRAVIS = "TRAVIS" in os.environ diff --git a/tests/test_new_csv_book.py b/tests/test_new_csv_book.py index 857f4ee..930a58a 100644 --- a/tests/test_new_csv_book.py +++ b/tests/test_new_csv_book.py @@ -3,13 +3,14 @@ from textwrap import dedent from unittest import TestCase import pyexcel_io.manager as manager -from nose.tools import raises from pyexcel_io._compact import OrderedDict from pyexcel_io.readers.tsv import TSVBookReader from pyexcel_io.writers.tsv import TSVBookWriter from pyexcel_io.readers.csvr import CSVBookReader from pyexcel_io.writers.csvw import CSVBookWriter +from nose.tools import raises + class TestCSVReaders(TestCase): file_type = "csv" diff --git a/tests/test_new_csvz_book.py b/tests/test_new_csvz_book.py index fdf8620..b73faed 100644 --- a/tests/test_new_csvz_book.py +++ b/tests/test_new_csvz_book.py @@ -5,7 +5,6 @@ import zipfile from unittest import TestCase import pyexcel_io.manager as manager -from nose.tools import raises from pyexcel_io import save_data from pyexcel_io._compact import OrderedDict from pyexcel_io.readers.csvz import CSVZipBookReader @@ -13,6 +12,8 @@ from pyexcel_io.readers.tsvz import TSVZipBookReader from pyexcel_io.writers.csvz import CSVZipBookWriter from pyexcel_io.writers.tsvz import TSVZipBookWriter +from nose.tools import raises + PY2 = sys.version_info[0] == 2 diff --git a/tests/test_pyexcel_integration.py b/tests/test_pyexcel_integration.py index e198c74..963da21 100644 --- a/tests/test_pyexcel_integration.py +++ b/tests/test_pyexcel_integration.py @@ -4,7 +4,6 @@ from textwrap import dedent from unittest import TestCase import pyexcel as pe - from pyexcel_io._compact import text_type diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 5be2174..209bad9 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -1,8 +1,9 @@ import os -from nose.tools import eq_ from pyexcel_io import get_data, save_data +from nose.tools import eq_ + class TestRenderer: def setUp(self): diff --git a/tests/test_service.py b/tests/test_service.py index cb6a7bd..5971708 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,5 +1,3 @@ -from nose import SkipTest -from nose.tools import eq_, raises from pyexcel_io.service import ( ODS_WRITE_FORMAT_COVERSION, date_value, @@ -12,6 +10,9 @@ from pyexcel_io.service import ( from pyexcel_io._compact import PY2 from pyexcel_io.exceptions import IntegerAccuracyLossError +from nose import SkipTest +from nose.tools import eq_, raises + def test_date_util_parse(): value = "2015-08-17T19:20:00" diff --git a/tests/test_sheet.py b/tests/test_sheet.py index 789317a..d82cb30 100644 --- a/tests/test_sheet.py +++ b/tests/test_sheet.py @@ -1,7 +1,8 @@ import pyexcel_io.constants as constants -from nose.tools import eq_ from pyexcel_io.sheet import SheetReader, SheetWriter +from nose.tools import eq_ + class MyWriter(SheetWriter): def set_size(self, size): diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 6927056..47bc5c9 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -3,6 +3,24 @@ import json import datetime import platform +from pyexcel_io._compact import OrderedDict +from pyexcel_io.database.common import ( + SQLTableExporter, + SQLTableImporter, + SQLTableExportAdapter, + SQLTableImportAdapter, +) +from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.database.exporters.sqlalchemy import ( + SQLBookReader, + SQLTableReader, +) +from pyexcel_io.database.importers.sqlalchemy import ( + SQLBookWriter, + SQLTableWriter, + PyexcelSQLSkipRowException, +) + from nose.tools import eq_, raises from sqlalchemy import ( Date, @@ -15,24 +33,7 @@ from sqlalchemy import ( create_engine, ) from sqlalchemy.orm import backref, relationship, sessionmaker -from pyexcel_io._compact import OrderedDict -from pyexcel_io.database.common import ( - SQLTableExporter, - SQLTableImporter, - SQLTableExportAdapter, - SQLTableImportAdapter, -) from sqlalchemy.ext.declarative import declarative_base -from pyexcel_io.database.querysets import QuerysetsReader -from pyexcel_io.database.exporters.sqlalchemy import ( - SQLBookReader, - SQLTableReader, -) -from pyexcel_io.database.importers.sqlalchemy import ( - SQLBookWriter, - SQLTableWriter, - PyexcelSQLSkipRowException, -) PY3 = sys.version_info[0] == 3 PY36 = PY3 and sys.version_info[1] == 6 From 2912ca6cb3d6b678c3889f0ad14761010df27eb9 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 22:22:00 +0100 Subject: [PATCH 059/143] :books: align ci system github does latest and travis does min --- .github/workflows/pythonpackage.yml | 29 +++++++++++++++++++++++++++++ .isort.cfg | 10 ++++++++++ .moban.d/custom_travis.yml.jj2 | 6 ++---- .travis.yml | 6 ++---- Makefile | 5 +++++ 5 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/pythonpackage.yml create mode 100644 .isort.cfg diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000..718696b --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,29 @@ +name: Python package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + make install_test format git-diff-check lint + - name: Test + run: | + make diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..d047cd9 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,10 @@ +[settings] +line_length=79 +known_first_party=lml, pyexcel +known_third_party=nose +indent=' ' +multi_line_output=3 +length_sort=1 +default_section=FIRSTPARTY +no_lines_before=LOCALFOLDER +sections=FUTURE,STDLIB,FIRSTPARTY,THIRDPARTY,LOCALFOLDER diff --git a/.moban.d/custom_travis.yml.jj2 b/.moban.d/custom_travis.yml.jj2 index 73b434a..9a18ae8 100644 --- a/.moban.d/custom_travis.yml.jj2 +++ b/.moban.d/custom_travis.yml.jj2 @@ -1,9 +1,7 @@ {% extends "travis.yml.jj2" %} {%block extra_matrix %} -matrix: - include: - - python: 3.6 - env: MINREQ=1 +env: + - MINREQ=1 {%endblock%} {%block custom_python_versions%} python: diff --git a/.travis.yml b/.travis.yml index 1012eba..208ab50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,8 @@ python: - 3.8 - 3.7 - 3.6 -matrix: - include: - - python: 3.6 - env: MINREQ=1 +env: + - MINREQ=1 stages: - lint diff --git a/Makefile b/Makefile index 48ac0a6..23f6e23 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ all: test test: bash test.sh +install_test: + pip install -r tests/requirements.txt + document: sphinx-autogen -o docs/source/generated/ docs/source/*.rst sphinx-build -b html docs/source/ docs/build/ @@ -15,4 +18,6 @@ format: lint: bash lint.sh +git-diff-check: + git diff --exit-code From 4ba1e4833ef8850e685da19ee70764dcabed7b54 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 22:25:57 +0100 Subject: [PATCH 060/143] :green_heart: make github ci pass --- .github/workflows/pythonpackage.yml | 1 + .moban.d/tests/custom-requirements.txt.jj2 | 1 + tests/requirements.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 718696b..a7a148a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -26,4 +26,5 @@ jobs: make install_test format git-diff-check lint - name: Test run: | + pip install -r tests/requirements.txt make diff --git a/.moban.d/tests/custom-requirements.txt.jj2 b/.moban.d/tests/custom-requirements.txt.jj2 index f5f7679..67d2e5d 100644 --- a/.moban.d/tests/custom-requirements.txt.jj2 +++ b/.moban.d/tests/custom-requirements.txt.jj2 @@ -6,4 +6,5 @@ pyexcel-xls>=0.1.0 moban black;python_version>="3.6" isort;python_version>="3.6" +collective.checkdocs {%endblock%} diff --git a/tests/requirements.txt b/tests/requirements.txt index 787faf3..f0b717a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,3 +9,4 @@ pyexcel-xls>=0.1.0 moban black;python_version>="3.6" isort;python_version>="3.6" +collective.checkdocs From 7acc301a97bed1b94c5b06a852833b84f527b029 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 23:09:05 +0100 Subject: [PATCH 061/143] :handshake: synchronize with pyexcel-mobans and pypi-mobans --- .github/workflows/pythonpackage.yml | 2 +- .moban.d/tests/custom-requirements.txt.jj2 | 10 ---------- .moban.yml | 1 - lint.sh | 3 ++- pyexcel-io.yml | 4 ++++ tests/requirements.txt | 12 ++++++------ 6 files changed, 13 insertions(+), 19 deletions(-) delete mode 100644 .moban.d/tests/custom-requirements.txt.jj2 diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a7a148a..7b71788 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -5,7 +5,7 @@ on: [push] jobs: build: - runs-on: ubuntu-latest + runs-on: windows-latest strategy: max-parallel: 4 matrix: diff --git a/.moban.d/tests/custom-requirements.txt.jj2 b/.moban.d/tests/custom-requirements.txt.jj2 deleted file mode 100644 index 67d2e5d..0000000 --- a/.moban.d/tests/custom-requirements.txt.jj2 +++ /dev/null @@ -1,10 +0,0 @@ -{% extends 'tests/requirements.txt.jj2' %} -{%block extras %} -SQLAlchemy -pyexcel>=0.2.0 -pyexcel-xls>=0.1.0 -moban -black;python_version>="3.6" -isort;python_version>="3.6" -collective.checkdocs -{%endblock%} diff --git a/.moban.yml b/.moban.yml index 44d697e..e6307f0 100644 --- a/.moban.yml +++ b/.moban.yml @@ -5,6 +5,5 @@ targets: - "docs/source/conf.py": "docs/source/custom_conf.py.jj2" - setup.py: io_setup.py.jj2 - .travis.yml: custom_travis.yml.jj2 - - "tests/requirements.txt": "tests/custom-requirements.txt.jj2" - README.rst: io_readme.rst.jj2 - "docs/source/index.rst": "docs/source/index.rst" diff --git a/lint.sh b/lint.sh index 976f745..6907d07 100644 --- a/lint.sh +++ b/lint.sh @@ -1,2 +1,3 @@ pip install flake8 -flake8 . --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long \ No newline at end of file +flake8 . --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long +python setup.py checkdocs diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 180d0f1..5b7f0cc 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -11,6 +11,10 @@ is_on_conda: true dependencies: - ordereddict;python_version<"2.7" - lml>=0.0.4 +test_dependencies: + - pyexcel + - pyexcel-xls + - SQLAlchemy extra_dependencies: - xls: - pyexcel-xls>=0.5.0 diff --git a/tests/requirements.txt b/tests/requirements.txt index f0b717a..1468098 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,10 +3,10 @@ mock;python_version<"3" codecov coverage flake8 -SQLAlchemy -pyexcel>=0.2.0 -pyexcel-xls>=0.1.0 -moban -black;python_version>="3.6" -isort;python_version>="3.6" +black +isort collective.checkdocs +moban +pyexcel +pyexcel-xls +SQLAlchemy From d643f72baea530dc64d1f2890563dfc89b1cd61f Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 23:12:31 +0100 Subject: [PATCH 062/143] :green_heart: pass lint --- tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index 1468098..ad5e0e3 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,6 +6,7 @@ flake8 black isort collective.checkdocs +pygments moban pyexcel pyexcel-xls From 87093eac389313ba1e032db28860109e37dcb735 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 23:16:43 +0100 Subject: [PATCH 063/143] :green_heart: pass github ci --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7b71788..cd563fd 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -27,4 +27,5 @@ jobs: - name: Test run: | pip install -r tests/requirements.txt + pip install -r rnd_requirements.txt make From 9455eb20090fc0f25d4611919cdfad71aceca319 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 23:19:45 +0100 Subject: [PATCH 064/143] :green_heart: pass github ci --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cd563fd..a9c85e1 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -5,7 +5,7 @@ on: [push] jobs: build: - runs-on: windows-latest + runs-on: ubuntu-latest strategy: max-parallel: 4 matrix: From 11c8f45eb63ae2d53022388c648f6839a188a39d Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 23:22:29 +0100 Subject: [PATCH 065/143] :bug: update write_data as store_data --- pyexcel_io/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index f20ed2c..ba3cb81 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -250,4 +250,4 @@ def get_writer( # backward compactibility -write_data = save_data +store_data = save_data From 4193e9b187c70494bbe7e73e37f716da2e24680a Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 23:30:00 +0100 Subject: [PATCH 066/143] :fire: remove rnd package --- .github/workflows/pythonpackage.yml | 1 - rnd_requirements.txt | 5 ----- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a9c85e1..a7a148a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -27,5 +27,4 @@ jobs: - name: Test run: | pip install -r tests/requirements.txt - pip install -r rnd_requirements.txt make diff --git a/rnd_requirements.txt b/rnd_requirements.txt index 37d8695..e69de29 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1,5 +0,0 @@ -. -https://github.com/chfw/lml/archive/master.zip -https://github.com/pyexcel/pyexcel/archive/dev.zip -https://github.com/pyexcel/pyexcel-xls/archive/dev.zip - From fc30cce46ea0a50560b83291a23dd2ee667f736a Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 12 Jun 2020 23:36:19 +0100 Subject: [PATCH 067/143] :hammer: code refactoring --- pyexcel-io.yml | 2 +- pyexcel_io/__init__.py | 2 +- pyexcel_io/_compact.py | 2 +- pyexcel_io/book.py | 2 +- pyexcel_io/database/__init__.py | 2 +- pyexcel_io/database/common.py | 2 +- pyexcel_io/database/exporters/django.py | 2 +- pyexcel_io/database/exporters/sqlalchemy.py | 2 +- pyexcel_io/database/importers/django.py | 2 +- pyexcel_io/database/importers/sqlalchemy.py | 2 +- pyexcel_io/database/querysets.py | 2 +- pyexcel_io/exceptions.py | 2 +- pyexcel_io/manager.py | 2 +- pyexcel_io/plugins.py | 2 +- pyexcel_io/readers/__init__.py | 2 +- pyexcel_io/readers/csvr.py | 4 +--- pyexcel_io/readers/csvz.py | 9 +++------ pyexcel_io/readers/tsv.py | 2 +- pyexcel_io/readers/tsvz.py | 2 +- pyexcel_io/service.py | 2 +- pyexcel_io/sheet.py | 2 +- pyexcel_io/utils.py | 2 +- pyexcel_io/writers/__init__.py | 2 +- pyexcel_io/writers/csvw.py | 2 +- pyexcel_io/writers/csvz.py | 2 +- pyexcel_io/writers/tsv.py | 2 +- pyexcel_io/writers/tsvz.py | 2 +- 27 files changed, 29 insertions(+), 34 deletions(-) diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 5b7f0cc..01e79e0 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -5,7 +5,7 @@ nick_name: io version: 0.6.0 current_version: 0.6.0 release: 0.5.20 -copyright_year: 2015-2019 +copyright_year: 2015-2020 moban_command: false is_on_conda: true dependencies: diff --git a/pyexcel_io/__init__.py b/pyexcel_io/__init__.py index 3362892..0e836b2 100644 --- a/pyexcel_io/__init__.py +++ b/pyexcel_io/__init__.py @@ -4,7 +4,7 @@ Uniform interface for reading/writing different excel file formats - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import logging diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index b0459f2..4e58bc3 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -4,7 +4,7 @@ Compatibles - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ # flake8: noqa diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index 46d9b87..d234329 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -4,7 +4,7 @@ The io interface to file extensions - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.manager as manager diff --git a/pyexcel_io/database/__init__.py b/pyexcel_io/database/__init__.py index addf50d..716c2e5 100644 --- a/pyexcel_io/database/__init__.py +++ b/pyexcel_io/database/__init__.py @@ -4,7 +4,7 @@ database data importer and exporter - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import IOPluginInfoChain diff --git a/pyexcel_io/database/common.py b/pyexcel_io/database/common.py index da46acd..b033fab 100644 --- a/pyexcel_io/database/common.py +++ b/pyexcel_io/database/common.py @@ -4,7 +4,7 @@ Common classes shared among database importers and exporters - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.book import BookReader diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index 933e587..d871a07 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -4,7 +4,7 @@ The lower level handler for django import and export - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.database.common import DbExporter diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index dc32ef8..dffebdd 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -4,7 +4,7 @@ The lower level handler for database import and export - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.database.common import DbExporter diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index dedc94a..d6b266d 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -4,7 +4,7 @@ The lower level handler for django import and export - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import logging diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 876d1b2..597db91 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -4,7 +4,7 @@ The lower level handler for database import and export - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index b062567..61640d4 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -4,7 +4,7 @@ The lower level handler for querysets - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import datetime diff --git a/pyexcel_io/exceptions.py b/pyexcel_io/exceptions.py index 458ad2c..8a8d1f6 100644 --- a/pyexcel_io/exceptions.py +++ b/pyexcel_io/exceptions.py @@ -4,7 +4,7 @@ all possible exceptions - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ diff --git a/pyexcel_io/manager.py b/pyexcel_io/manager.py index fe3a23a..452ef18 100644 --- a/pyexcel_io/manager.py +++ b/pyexcel_io/manager.py @@ -4,7 +4,7 @@ Control file streams - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io._compact import BytesIO, StringIO diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 2c350ef..102445f 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -4,7 +4,7 @@ factory for getting readers and writers - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.utils as ioutils diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index 754e022..3a4b5dc 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -4,7 +4,7 @@ file readers - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import IOPluginInfoChain diff --git a/pyexcel_io/readers/csvr.py b/pyexcel_io/readers/csvr.py index e2cc20a..12f89ef 100644 --- a/pyexcel_io/readers/csvr.py +++ b/pyexcel_io/readers/csvr.py @@ -4,7 +4,7 @@ csv file reader - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import os @@ -130,8 +130,6 @@ class CSVSheetReader(SheetReader): def column_iterator(self, row): for element in row: - if compact.PY2: - element = element.decode("utf-8") if element is not None and element != "": element = self.__convert_cell(element) yield element diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index c956eea..173c5d7 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -4,13 +4,13 @@ The lower level csvz file format handler. - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import zipfile from pyexcel_io.book import BookReader -from pyexcel_io._compact import PY2, StringIO +from pyexcel_io._compact import StringIO from pyexcel_io.constants import FILE_FORMAT_CSVZ from .csvr import NamedContent, CSVinMemoryReader @@ -40,10 +40,7 @@ class CSVZipBookReader(BookReader): def read_sheet(self, native_sheet): content = self.zipfile.read(native_sheet.payload) - if PY2: - sheet = StringIO(content) - else: - sheet = StringIO(content.decode("utf-8")) + sheet = StringIO(content.decode("utf-8")) reader = CSVinMemoryReader( NamedContent(native_sheet.name, sheet), **self._keywords diff --git a/pyexcel_io/readers/tsv.py b/pyexcel_io/readers/tsv.py index 35829ae..abf7802 100644 --- a/pyexcel_io/readers/tsv.py +++ b/pyexcel_io/readers/tsv.py @@ -4,7 +4,7 @@ The lower level tsv file format handler. - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py index 9dc7d6d..1e6848a 100644 --- a/pyexcel_io/readers/tsvz.py +++ b/pyexcel_io/readers/tsvz.py @@ -4,7 +4,7 @@ The lower level tsvz file format handler. - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT diff --git a/pyexcel_io/service.py b/pyexcel_io/service.py index 5973ae3..dcc92de 100644 --- a/pyexcel_io/service.py +++ b/pyexcel_io/service.py @@ -4,7 +4,7 @@ provide service code to downstream projects - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import re diff --git a/pyexcel_io/sheet.py b/pyexcel_io/sheet.py index 3134b8e..6f27fc7 100644 --- a/pyexcel_io/sheet.py +++ b/pyexcel_io/sheet.py @@ -4,7 +4,7 @@ The io interface to file extensions - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants diff --git a/pyexcel_io/utils.py b/pyexcel_io/utils.py index 2207e7f..c1d6c94 100644 --- a/pyexcel_io/utils.py +++ b/pyexcel_io/utils.py @@ -4,7 +4,7 @@ utility functions - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants diff --git a/pyexcel_io/writers/__init__.py b/pyexcel_io/writers/__init__.py index 30f62eb..f85263a 100644 --- a/pyexcel_io/writers/__init__.py +++ b/pyexcel_io/writers/__init__.py @@ -4,7 +4,7 @@ file writers - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import IOPluginInfoChain diff --git a/pyexcel_io/writers/csvw.py b/pyexcel_io/writers/csvw.py index ff4ead1..e7036e0 100644 --- a/pyexcel_io/writers/csvw.py +++ b/pyexcel_io/writers/csvw.py @@ -4,7 +4,7 @@ The lower level csv file format writer - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import csv diff --git a/pyexcel_io/writers/csvz.py b/pyexcel_io/writers/csvz.py index 381168f..e4905d6 100644 --- a/pyexcel_io/writers/csvz.py +++ b/pyexcel_io/writers/csvz.py @@ -4,7 +4,7 @@ The lower level csvz file format handler. - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import zipfile diff --git a/pyexcel_io/writers/tsv.py b/pyexcel_io/writers/tsv.py index 757cad2..33f6577 100644 --- a/pyexcel_io/writers/tsv.py +++ b/pyexcel_io/writers/tsv.py @@ -4,7 +4,7 @@ The lower level tsv file format handler. - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants diff --git a/pyexcel_io/writers/tsvz.py b/pyexcel_io/writers/tsvz.py index b12814d..1528e4f 100644 --- a/pyexcel_io/writers/tsvz.py +++ b/pyexcel_io/writers/tsvz.py @@ -4,7 +4,7 @@ The lower level tsvz file format handler. - :copyright: (c) 2014-2017 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT From ca35d108d754953344b48188f91f69807ae218b0 Mon Sep 17 00:00:00 2001 From: jaska Date: Fri, 19 Jun 2020 22:46:02 +0100 Subject: [PATCH 068/143] New style plugin - reader (#73) * :hammer: intial draft of the new reader * :hammer: rewrote tsv reader * :hammer: rewrite tsvz csvz readers * :hammer: isolate what pyexcel-io knows about the sheet and plugin knows about the sheet --- pyexcel_io/io.py | 53 +++++---- pyexcel_io/plugins.py | 80 ++++++++++++- pyexcel_io/reader.py | 118 +++++++++++++++++-- pyexcel_io/readers/__init__.py | 46 +++++++- pyexcel_io/readers/csv_content_reader.py | 21 ++++ pyexcel_io/readers/csv_file_reader.py | 63 ++++++++++ pyexcel_io/readers/csv_memory_reader.py | 59 ++++++++++ pyexcel_io/readers/csvr.py | 139 +---------------------- pyexcel_io/readers/csvz.py | 59 ++++------ pyexcel_io/readers/tsv.py | 35 ++++-- pyexcel_io/readers/tsvz.py | 16 +-- pyexcel_io/sheet.py | 4 +- pyexcel_io/utils.py | 3 +- tests/test_csv_book.py | 21 +++- tests/test_new_csv_book.py | 52 +++++++-- tests/test_new_csvz_book.py | 20 ++-- 16 files changed, 526 insertions(+), 263 deletions(-) create mode 100644 pyexcel_io/readers/csv_content_reader.py create mode 100644 pyexcel_io/readers/csv_file_reader.py create mode 100644 pyexcel_io/readers/csv_memory_reader.py diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index ba3cb81..a02fc81 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -14,8 +14,12 @@ from types import GeneratorType from pyexcel_io import constants from pyexcel_io.reader import Reader from pyexcel_io.writer import Writer +from pyexcel_io.plugins import OLD_READERS from pyexcel_io._compact import isstream -from pyexcel_io.exceptions import NoSupportingPluginFound +from pyexcel_io.exceptions import ( + NoSupportingPluginFound, + SupportingPluginAvailableButNotInstalled, +) def iget_data(afile, file_type=None, **keywords): @@ -169,7 +173,32 @@ def load_data( raise Exception(constants.MESSAGE_FILE_NAME_SHOULD_BE_STRING) try: + reader = OLD_READERS.get_a_plugin(file_type, library) + except (NoSupportingPluginFound, SupportingPluginAvailableButNotInstalled): reader = Reader(file_type, library) + + try: + if file_name: + reader.open(file_name, **keywords) + elif file_content: + reader.open_content(file_content, **keywords) + elif file_stream: + reader.open_stream(file_stream, **keywords) + if sheet_name: + result = reader.read_sheet_by_name(sheet_name) + elif sheet_index is not None: + result = reader.read_sheet_by_index(sheet_index) + elif sheets is not None: + result = reader.read_many(sheets) + else: + result = reader.read_all() + if streaming is False: + for key in result.keys(): + result[key] = list(result[key]) + reader.close() + reader = None + + return result, reader except NoSupportingPluginFound: if file_name: if os.path.exists(file_name): @@ -186,28 +215,6 @@ def load_data( else: raise - if file_name: - reader.open(file_name, **keywords) - elif file_content: - reader.open_content(file_content, **keywords) - elif file_stream: - reader.open_stream(file_stream, **keywords) - if sheet_name: - result = reader.read_sheet_by_name(sheet_name) - elif sheet_index is not None: - result = reader.read_sheet_by_index(sheet_index) - elif sheets is not None: - result = reader.read_many(sheets) - else: - result = reader.read_all() - if streaming is False: - for key in result.keys(): - result[key] = list(result[key]) - reader.close() - reader = None - - return result, reader - def get_writer( file_name=None, diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 102445f..c403947 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -18,6 +18,7 @@ ERROR_MESSAGE_FORMATTER = "one of these plugins for %s data in '%s': %s" UPGRADE_MESSAGE = "Please upgrade the plugin '%s' according to \ plugin compactibility table." READER_PLUGIN = "pyexcel-io reader" +NEW_READER_PLUGIN = "pyexcel-io new reader" WRITER_PLUGIN = "pyexcel-io writer" @@ -63,6 +64,26 @@ class IOPluginInfoChain(PluginInfoChain): return self.add_a_plugin_instance(a_plugin_info) +class NewIOPluginInfoChain(PluginInfoChain): + """provide custom functions to add a reader and a writer """ + + def add_a_reader( + self, + relative_plugin_class_path=None, + location="file", + file_types=None, + stream_type=None, + ): + """ add pyexcle-io reader plugin info """ + a_plugin_info = IOPluginInfo( + NEW_READER_PLUGIN, + self._get_abs_path(relative_plugin_class_path), + file_types=[f"{location}-{file_type}" for file_type in file_types], + stream_type=stream_type, + ) + return self.add_a_plugin_instance(a_plugin_info) + + class IOManager(PluginManager): """Manage pyexcel-io plugins""" @@ -119,14 +140,71 @@ class IOManager(PluginManager): return all_formats +class NewIOManager(IOManager): + def load_me_later(self, plugin_info): + PluginManager.load_me_later(self, plugin_info) + _do_additional_registration(plugin_info) + + def register_a_plugin(self, cls, plugin_info): + """ for dynamically loaded plugin """ + PluginManager.register_a_plugin(self, cls, plugin_info) + _do_additional_registration(plugin_info) + + def get_a_plugin( + self, file_type=None, location=None, library=None, **keywords + ): + __file_type = file_type.lower() + plugin = self.load_me_now(f"{location}-{__file_type}", library=library) + handler = plugin() + handler.set_type(__file_type) + return handler + + def raise_exception(self, file_type): + file_type = file_type.split("-")[1] + plugins = self.known_plugins.get(file_type, None) + if plugins: + message = "Please install " + if len(plugins) > 1: + message += ERROR_MESSAGE_FORMATTER % ( + self.action, + file_type, + ",".join(plugins), + ) + else: + message += plugins[0] + raise exceptions.SupportingPluginAvailableButNotInstalled(message) + + else: + raise exceptions.NoSupportingPluginFound( + "No suitable library found for %s" % file_type + ) + + def get_all_formats(self): + """ return all supported formats """ + all_formats = set( + [x.split("-")[1] for x in self.registry.keys()] + + list(self.known_plugins.keys()) + ) + return all_formats + + def _do_additional_registration(plugin_info): for file_type in plugin_info.tags(): manager.register_stream_type(file_type, plugin_info.stream_type) manager.register_a_file_type(file_type, plugin_info.stream_type, None) -READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS) +class FakeReaders: + def get_all_formats(self): + return OLD_READERS.get_all_formats().union( + NEW_READERS.get_all_formats() + ) + + +OLD_READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS) WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS) +NEW_READERS = NewIOManager(NEW_READER_PLUGIN, ioutils.AVAILABLE_READERS) +READERS = FakeReaders() def load_plugins(plugin_name_patterns, path, black_list, white_list): diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py index 1161352..1103938 100644 --- a/pyexcel_io/reader.py +++ b/pyexcel_io/reader.py @@ -1,30 +1,116 @@ -from pyexcel_io.plugins import READERS +from pyexcel_io import exceptions +from pyexcel_io.book import _convert_content_to_stream +from pyexcel_io.sheet import SheetReader +from pyexcel_io.plugins import NEW_READERS +from pyexcel_io._compact import OrderedDict + + +def clean_keywords(keywords): + sheet_keywords = {} + native_sheet_keywords = {} + args_list = [ + "start_row", + "row_limit", + "start_column", + "column_limit", + "skip_column_func", + "skip_row_func", + "skip_empty_rows", + "row_renderer", + ] + for arg in keywords: + if arg in args_list: + sheet_keywords[arg] = keywords[arg] + else: + native_sheet_keywords[arg] = keywords[arg] + return sheet_keywords, native_sheet_keywords class Reader(object): def __init__(self, file_type, library=None): - self.reader = READERS.get_a_plugin(file_type, library) + self.file_type = file_type + self.library = library + self.keywords = None def open(self, file_name, **keywords): - return self.reader.open(file_name, **keywords) + self.reader = NEW_READERS.get_a_plugin( + self.file_type, location="file", library=self.library + ) + self.keywords, native_sheet_keywords = clean_keywords(keywords) + return self.reader.open(file_name, **native_sheet_keywords) def open_content(self, file_content, **keywords): - return self.reader.open_content(file_content, **keywords) + self.keywords, native_sheet_keywords = clean_keywords(keywords) + try: + self.reader = NEW_READERS.get_a_plugin( + self.file_type, location="content", library=self.library + ) + return self.reader.open(file_content, **native_sheet_keywords) + except ( + exceptions.NoSupportingPluginFound, + exceptions.SupportingPluginAvailableButNotInstalled, + ): + file_stream = _convert_content_to_stream( + file_content, self.file_type + ) + return self.open_stream(file_stream, **native_sheet_keywords) def open_stream(self, file_stream, **keywords): - return self.reader.open_stream(file_stream, **keywords) - - def read_sheet_by_index(self, sheet_index): - return self.reader.read_sheet_by_index(sheet_index) + self.keywords, native_sheet_keywords = clean_keywords(keywords) + self.reader = NEW_READERS.get_a_plugin( + self.file_type, location="memory", library=self.library + ) + return self.reader.open(file_stream, **native_sheet_keywords) def read_sheet_by_name(self, sheet_name): - return self.reader.read_sheet_by_name(sheet_name) + """ + read a named sheet from a excel data book + """ + for index, content in enumerate(self.reader.content_array): + if content.name == sheet_name: + return {content.name: self.read_sheet(index)} + else: + raise ValueError("Cannot find sheet %s" % sheet_name) - def read_many(self, sheets): - return self.reader.read_many(sheets) + def read_sheet(self, sheet_index): + sheet_reader = self.reader.read_sheet(sheet_index) + sheet = EncapsulatedSheetReader(sheet_reader, **self.keywords) + return sheet.to_array() + + def read_sheet_by_index(self, sheet_index): + """ + read an indexed sheet from a excel data book + """ + try: + name = self.reader.content_array[sheet_index].name + return {name: self.read_sheet(sheet_index)} + + except IndexError: + self.close() + raise def read_all(self): - return self.reader.read_all() + """ + read everything from a excel data book + """ + result = OrderedDict() + for index, sheet in enumerate(self.reader.content_array): + result.update( + {self.reader.content_array[index].name: self.read_sheet(index)} + ) + return result + + def read_many(self, sheets): + """ + read everything from a excel data book + """ + result = OrderedDict() + for sheet in sheets: + if isinstance(sheet, int): + result.update(self.read_sheet_by_index(sheet)) + else: + result.update(self.read_sheet_by_name(sheet)) + return result def close(self): return self.reader.close() @@ -34,3 +120,11 @@ class Reader(object): def __exit__(self, a_type, value, traceback): self.close() + + +class EncapsulatedSheetReader(SheetReader): + def row_iterator(self): + yield from self._native_sheet.row_iterator() + + def column_iterator(self, row): + yield from self._native_sheet.column_iterator(row) diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index 3a4b5dc..c302e64 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -7,22 +7,56 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.plugins import IOPluginInfoChain +from pyexcel_io.plugins import IOPluginInfoChain, NewIOPluginInfoChain -IOPluginInfoChain(__name__).add_a_reader( - relative_plugin_class_path="csvr.CSVBookReader", +NewIOPluginInfoChain(__name__).add_a_reader( + relative_plugin_class_path="csv_file_reader.FileReader", + location="file", file_types=["csv"], stream_type="text", ).add_a_reader( - relative_plugin_class_path="tsv.TSVBookReader", + relative_plugin_class_path="csv_content_reader.ContentReader", + location="content", + file_types=["csv"], + stream_type="text", +).add_a_reader( + relative_plugin_class_path="csv_memory_reader.MemoryReader", + location="memory", + file_types=["csv"], + stream_type="text", +).add_a_reader( + relative_plugin_class_path="tsv.TSVMemoryReader", + location="memory", file_types=["tsv"], stream_type="text", ).add_a_reader( - relative_plugin_class_path="csvz.CSVZipBookReader", + relative_plugin_class_path="tsv.TSVFileReader", + location="file", + file_types=["tsv"], + stream_type="text", +).add_a_reader( + relative_plugin_class_path="tsv.TSVContentReader", + location="content", + file_types=["tsv"], + stream_type="text", +).add_a_reader( + relative_plugin_class_path="csvz.FileReader", file_types=["csvz"], + location="file", stream_type="binary", ).add_a_reader( - relative_plugin_class_path="tsvz.TSVZipBookReader", + relative_plugin_class_path="csvz.FileReader", + file_types=["csvz"], + location="memory", + stream_type="binary", +).add_a_reader( + relative_plugin_class_path="tsvz.TSVZipFileReader", file_types=["tsvz"], + location="file", + stream_type="binary", +).add_a_reader( + relative_plugin_class_path="tsvz.TSVZipFileReader", + file_types=["tsvz"], + location="memory", stream_type="binary", ) diff --git a/pyexcel_io/readers/csv_content_reader.py b/pyexcel_io/readers/csv_content_reader.py new file mode 100644 index 0000000..52f29dc --- /dev/null +++ b/pyexcel_io/readers/csv_content_reader.py @@ -0,0 +1,21 @@ +import mmap + +from pyexcel_io.book import _convert_content_to_stream +from pyexcel_io.readers.csvr import CSVMemoryMapIterator +from pyexcel_io.readers.csv_memory_reader import MemoryReader + + +class ContentReader(MemoryReader): + def open(self, file_content, **keywords): + encoding = keywords.get("encoding", "utf-8") + if isinstance(file_content, mmap.mmap): + # load from mmap + file_stream = CSVMemoryMapIterator(file_content, encoding) + else: + if isinstance(file_content, bytes): + file_content = file_content.decode(encoding) + + file_stream = _convert_content_to_stream( + file_content, self.file_type + ) + super(ContentReader, self).open(file_stream, **keywords) diff --git a/pyexcel_io/readers/csv_file_reader.py b/pyexcel_io/readers/csv_file_reader.py new file mode 100644 index 0000000..5f196f0 --- /dev/null +++ b/pyexcel_io/readers/csv_file_reader.py @@ -0,0 +1,63 @@ +import os +import re +import glob + +from pyexcel_io import constants +from pyexcel_io.sheet import NamedContent +from pyexcel_io.readers.csvr import CSVFileReader + +DEFAULT_NEWLINE = "\r\n" + + +class FileReader(object): + def __init__(self): + self.handles = [] + + def set_type(self, _): + pass + + def open(self, file_name, **keywords): + """Load content from a file + :params str filename: an accessible file path + :returns: a book + """ + self.keywords = keywords + self.__line_terminator = keywords.get( + constants.KEYWORD_LINE_TERMINATOR, DEFAULT_NEWLINE + ) + names = os.path.splitext(file_name) + filepattern = "%s%s*%s*%s" % ( + names[0], + constants.DEFAULT_MULTI_CSV_SEPARATOR, + constants.DEFAULT_MULTI_CSV_SEPARATOR, + names[1], + ) + filelist = glob.glob(filepattern) + if len(filelist) == 0: + file_parts = os.path.split(file_name) + self.content_array = [NamedContent(file_parts[-1], file_name)] + + else: + matcher = "%s%s(.*)%s(.*)%s" % ( + names[0], + constants.DEFAULT_MULTI_CSV_SEPARATOR, + constants.DEFAULT_MULTI_CSV_SEPARATOR, + names[1], + ) + tmp_file_list = [] + for filen in filelist: + result = re.match(matcher, filen) + tmp_file_list.append((result.group(1), result.group(2), filen)) + ret = [] + for lsheetname, index, filen in sorted( + tmp_file_list, key=lambda row: row[1] + ): + ret.append(NamedContent(lsheetname, filen)) + self.content_array = ret + + def read_sheet(self, index): + return CSVFileReader(self.content_array[index], **self.keywords) + + def close(self): + for reader in self.handles: + reader.close() diff --git a/pyexcel_io/readers/csv_memory_reader.py b/pyexcel_io/readers/csv_memory_reader.py new file mode 100644 index 0000000..830d0f2 --- /dev/null +++ b/pyexcel_io/readers/csv_memory_reader.py @@ -0,0 +1,59 @@ +import re + +import pyexcel_io._compact as compact +from pyexcel_io import constants +from pyexcel_io.sheet import NamedContent +from pyexcel_io.readers.csvr import CSVinMemoryReader + +DEFAULT_SHEET_SEPARATOR_FORMATTER = f"---{constants.DEFAULT_NAME}---%s" + + +class MemoryReader(object): + def __init__(self): + self.handles = [] + self.file_type = constants.FILE_FORMAT_CSV + + def set_type(self, _): + pass + + def open(self, file_stream, multiple_sheets=False, **keywords): + """Load content from memory + :params stream file_content: the actual file content in memory + :returns: a book + """ + self.keywords = keywords + self.__load_from_memory_flag = True + self.__line_terminator = keywords.get( + constants.KEYWORD_LINE_TERMINATOR, constants.DEFAULT_CSV_NEWLINE + ) + separator = DEFAULT_SHEET_SEPARATOR_FORMATTER % self.__line_terminator + if multiple_sheets: + # will be slow for large files + file_stream.seek(0) + content = file_stream.read() + sheets = content.split(separator) + named_contents = [] + for sheet in sheets: + if sheet == "": # skip empty named sheet + continue + + lines = sheet.split(self.__line_terminator) + result = re.match(constants.SEPARATOR_MATCHER, lines[0]) + new_content = "\n".join(lines[1:]) + new_sheet = NamedContent( + result.group(1), compact.StringIO(new_content) + ) + named_contents.append(new_sheet) + self.content_array = named_contents + + else: + if hasattr(file_stream, "seek"): + file_stream.seek(0) + self.content_array = [NamedContent(self.file_type, file_stream)] + + def read_sheet(self, index): + return CSVinMemoryReader(self.content_array[index], **self.keywords) + + def close(self): + for reader in self.handles: + reader.close() diff --git a/pyexcel_io/readers/csvr.py b/pyexcel_io/readers/csvr.py index 12f89ef..9eb72a4 100644 --- a/pyexcel_io/readers/csvr.py +++ b/pyexcel_io/readers/csvr.py @@ -7,16 +7,11 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -import os -import re import csv -import glob import pyexcel_io.service as service import pyexcel_io._compact as compact import pyexcel_io.constants as constants -from pyexcel_io.book import BookReader -from pyexcel_io.sheet import SheetReader, NamedContent DEFAULT_SEPARATOR = "__" DEFAULT_SHEET_SEPARATOR_FORMATTER = "---%s---" % constants.DEFAULT_NAME + "%s" @@ -92,8 +87,11 @@ class CSVMemoryMapIterator(compact.Iterator): return line + def close(self): + pass -class CSVSheetReader(SheetReader): + +class CSVSheetReader(object): """ generic csv file reader""" def __init__( @@ -109,7 +107,8 @@ class CSVSheetReader(SheetReader): default_float_nan=None, **keywords ): - SheetReader.__init__(self, sheet, **keywords) + self._native_sheet = sheet + self._keywords = keywords self._encoding = encoding self.__auto_detect_int = auto_detect_int self.__auto_detect_float = auto_detect_float @@ -190,129 +189,3 @@ class CSVinMemoryReader(CSVSheetReader): unicode_reader = self._native_sheet.payload return unicode_reader - - -class CSVBookReader(BookReader): - """ read csv file """ - - def __init__(self): - BookReader.__init__(self) - self._file_type = constants.FILE_FORMAT_CSV - self._file_content = None - self.__load_from_memory_flag = False - self.__line_terminator = constants.DEFAULT_CSV_NEWLINE - self.__sheet_name = None - self.__sheet_index = None - self.__multiple_sheets = False - self.__readers = [] - - def open(self, file_name, **keywords): - BookReader.open(self, file_name, **keywords) - self._native_book = self._load_from_file() - - def open_stream(self, file_stream, multiple_sheets=False, **keywords): - BookReader.open_stream(self, file_stream, **keywords) - self.__multiple_sheets = multiple_sheets - self._native_book = self._load_from_stream() - - def open_content(self, file_content, **keywords): - import mmap - - encoding = keywords.get("encoding", "utf-8") - if isinstance(file_content, mmap.mmap): - # load from mmap - self.__multiple_sheets = keywords.get("multiple_sheets", False) - self._file_stream = CSVMemoryMapIterator(file_content, encoding) - self._keywords = keywords - self._native_book = self._load_from_stream() - else: - if isinstance(file_content, bytes): - file_content = file_content.decode(encoding) - - BookReader.open_content(self, file_content, **keywords) - - def read_sheet(self, native_sheet): - if self.__load_from_memory_flag: - reader = CSVinMemoryReader(native_sheet, **self._keywords) - else: - reader = CSVFileReader(native_sheet, **self._keywords) - self.__readers.append(reader) - return reader.to_array() - - def close(self): - for reader in self.__readers: - reader.close() - - def _load_from_stream(self): - """Load content from memory - - :params stream file_content: the actual file content in memory - :returns: a book - """ - self.__load_from_memory_flag = True - self.__line_terminator = self._keywords.get( - constants.KEYWORD_LINE_TERMINATOR, self.__line_terminator - ) - separator = DEFAULT_SHEET_SEPARATOR_FORMATTER % self.__line_terminator - if self.__multiple_sheets: - # will be slow for large files - self._file_stream.seek(0) - content = self._file_stream.read() - sheets = content.split(separator) - named_contents = [] - for sheet in sheets: - if sheet == "": # skip empty named sheet - continue - - lines = sheet.split(self.__line_terminator) - result = re.match(constants.SEPARATOR_MATCHER, lines[0]) - new_content = "\n".join(lines[1:]) - new_sheet = NamedContent( - result.group(1), compact.StringIO(new_content) - ) - named_contents.append(new_sheet) - return named_contents - - else: - if hasattr(self._file_stream, "seek"): - self._file_stream.seek(0) - return [NamedContent(self._file_type, self._file_stream)] - - def _load_from_file(self): - """Load content from a file - - :params str filename: an accessible file path - :returns: a book - """ - self.__line_terminator = self._keywords.get( - constants.KEYWORD_LINE_TERMINATOR, self.__line_terminator - ) - names = os.path.splitext(self._file_name) - filepattern = "%s%s*%s*%s" % ( - names[0], - constants.DEFAULT_MULTI_CSV_SEPARATOR, - constants.DEFAULT_MULTI_CSV_SEPARATOR, - names[1], - ) - filelist = glob.glob(filepattern) - if len(filelist) == 0: - file_parts = os.path.split(self._file_name) - return [NamedContent(file_parts[-1], self._file_name)] - - else: - matcher = "%s%s(.*)%s(.*)%s" % ( - names[0], - constants.DEFAULT_MULTI_CSV_SEPARATOR, - constants.DEFAULT_MULTI_CSV_SEPARATOR, - names[1], - ) - tmp_file_list = [] - for filen in filelist: - result = re.match(matcher, filen) - tmp_file_list.append((result.group(1), result.group(2), filen)) - ret = [] - for lsheetname, index, filen in sorted( - tmp_file_list, key=lambda row: row[1] - ): - ret.append(NamedContent(lsheetname, filen)) - return ret diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index 173c5d7..829a733 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -9,61 +9,44 @@ """ import zipfile -from pyexcel_io.book import BookReader +from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import StringIO -from pyexcel_io.constants import FILE_FORMAT_CSVZ - -from .csvr import NamedContent, CSVinMemoryReader +from pyexcel_io.readers.csvr import CSVinMemoryReader -class CSVZipBookReader(BookReader): - """csvz reader - - Read zipped csv file that was zipped up by pyexcel-io. It support - single csv file and multiple csv files. - """ - +class FileReader(object): def __init__(self): - BookReader.__init__(self) - self._file_type = FILE_FORMAT_CSVZ - self.zipfile = None + self.content_array = [] + self.keywords = None - def open(self, file_name, **keywords): - BookReader.open(self, file_name, **keywords) - self._native_book = self._load_from_file_alike_object(self._file_name) + def set_type(self, _): + pass - def open_stream(self, file_stream, **keywords): - BookReader.open_stream(self, file_stream, **keywords) - self._native_book = self._load_from_file_alike_object( - self._file_stream - ) - - def read_sheet(self, native_sheet): - content = self.zipfile.read(native_sheet.payload) - sheet = StringIO(content.decode("utf-8")) - - reader = CSVinMemoryReader( - NamedContent(native_sheet.name, sheet), **self._keywords - ) - return reader.to_array() - - def close(self): - if self.zipfile: - self.zipfile.close() - - def _load_from_file_alike_object(self, file_alike_object): + def open(self, file_alike_object, **keywords): try: self.zipfile = zipfile.ZipFile(file_alike_object, "r") sheets = [ NamedContent(_get_sheet_name(name), name) for name in self.zipfile.namelist() ] - return sheets + self.content_array = sheets + self.keywords = keywords except zipfile.BadZipfile: print("StringIO instance was passed by any chance?") raise + def close(self): + if self.zipfile: + self.zipfile.close() + + def read_sheet(self, index): + name = self.content_array[index].name + content = self.zipfile.read(self.content_array[index].payload) + sheet = StringIO(content.decode("utf-8")) + + return CSVinMemoryReader(NamedContent(name, sheet), **self.keywords) + def _get_sheet_name(filename): len_of_a_dot = 1 diff --git a/pyexcel_io/readers/tsv.py b/pyexcel_io/readers/tsv.py index abf7802..d505994 100644 --- a/pyexcel_io/readers/tsv.py +++ b/pyexcel_io/readers/tsv.py @@ -8,21 +8,32 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants - -from .csvr import CSVBookReader +from pyexcel_io.readers.csv_file_reader import FileReader +from pyexcel_io.readers.csv_memory_reader import MemoryReader +from pyexcel_io.readers.csv_content_reader import ContentReader -class TSVBookReader(CSVBookReader): - """ Read tab separated values """ - - def __init__(self): - CSVBookReader.__init__(self) - self._file_type = constants.FILE_FORMAT_TSV - +class TSVFileReader(FileReader): def open(self, file_name, **keywords): keywords["dialect"] = constants.KEYWORD_TSV_DIALECT - CSVBookReader.open(self, file_name, **keywords) + super(TSVFileReader, self).open(file_name, **keywords) - def open_stream(self, file_content, **keywords): + +class TSVMemoryReader(MemoryReader): + def __init__(self): + self.handles = [] + self.file_type = constants.FILE_FORMAT_TSV + + def open(self, file_stream, **keywords): keywords["dialect"] = constants.KEYWORD_TSV_DIALECT - CSVBookReader.open_stream(self, file_content, **keywords) + super(TSVMemoryReader, self).open(file_stream, **keywords) + + +class TSVContentReader(ContentReader): + def __init__(self): + self.handles = [] + self.file_type = constants.FILE_FORMAT_TSV + + def open(self, file_content, **keywords): + keywords["dialect"] = constants.KEYWORD_TSV_DIALECT + super(TSVContentReader, self).open(file_content, **keywords) diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py index 1e6848a..3ca5e91 100644 --- a/pyexcel_io/readers/tsvz.py +++ b/pyexcel_io/readers/tsvz.py @@ -7,25 +7,17 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT +from pyexcel_io.constants import KEYWORD_TSV_DIALECT -from .csvz import CSVZipBookReader +from .csvz import FileReader -class TSVZipBookReader(CSVZipBookReader): +class TSVZipFileReader(FileReader): """ read zipped tab separated value file it supports single tsv file and mulitple tsv files """ - def __init__(self): - CSVZipBookReader.__init__(self) - self._file_type = FILE_FORMAT_TSVZ - def open(self, file_name, **keywords): keywords["dialect"] = KEYWORD_TSV_DIALECT - CSVZipBookReader.open(self, file_name, **keywords) - - def open_stream(self, file_content, **keywords): - keywords["dialect"] = KEYWORD_TSV_DIALECT - CSVZipBookReader.open_stream(self, file_content, **keywords) + super(TSVZipFileReader, self).open(file_name, **keywords) diff --git a/pyexcel_io/sheet.py b/pyexcel_io/sheet.py index 6f27fc7..7d54b57 100644 --- a/pyexcel_io/sheet.py +++ b/pyexcel_io/sheet.py @@ -38,11 +38,11 @@ class SheetReader(object): skip_column_func=None, skip_empty_rows=False, row_renderer=None, - **keywords + **deprecated_use_of_keywords_here ): self._native_sheet = sheet self._keywords = {} - self._keywords.update(keywords) + self._keywords.update(deprecated_use_of_keywords_here) self._start_row = start_row self._row_limit = row_limit self._start_column = start_column diff --git a/pyexcel_io/utils.py b/pyexcel_io/utils.py index c1d6c94..f25f7ab 100644 --- a/pyexcel_io/utils.py +++ b/pyexcel_io/utils.py @@ -16,13 +16,14 @@ ODS3_PLUGIN = "pyexcel-ods3" XLSXW_PLUGIN = "pyexcel-xlsxw" IO_ITSELF = "pyexcel-io" +AVAILABLE_NEW_READERS = {} AVAILABLE_READERS = { + constants.FILE_FORMAT_CSV: [IO_ITSELF], constants.FILE_FORMAT_XLS: [XLS_PLUGIN], constants.FILE_FORMAT_XLSX: [XLS_PLUGIN, XLSX_PLUGIN], constants.FILE_FORMAT_XLSM: [XLS_PLUGIN, XLSX_PLUGIN], constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN], - constants.FILE_FORMAT_CSV: [IO_ITSELF], constants.FILE_FORMAT_TSV: [IO_ITSELF], constants.FILE_FORMAT_CSVZ: [IO_ITSELF], constants.FILE_FORMAT_TSVZ: [IO_ITSELF], diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index ecbacb6..74a31c1 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -7,6 +7,7 @@ from unittest import TestCase import pyexcel_io.manager as manager from pyexcel_io.sheet import NamedContent +from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import PY2, BytesIO, StringIO from pyexcel_io.readers.csvr import ( CSVFileReader, @@ -34,7 +35,9 @@ class TestReaders(TestCase): sheet.get_file_handle() def test_sheet_file_reader(self): - r = CSVFileReader(NamedContent(self.file_type, self.test_file)) + r = EncapsulatedSheetReader( + CSVFileReader(NamedContent(self.file_type, self.test_file)) + ) result = list(r.to_array()) self.assertEqual(result, self.expected_data) @@ -43,7 +46,9 @@ class TestReaders(TestCase): with open(self.test_file, "r") as f: io.write(f.read()) io.seek(0) - r = CSVinMemoryReader(NamedContent(self.file_type, io)) + r = EncapsulatedSheetReader( + CSVinMemoryReader(NamedContent(self.file_type, io)) + ) result = list(r.to_array()) self.assertEqual(result, self.expected_data) @@ -110,7 +115,9 @@ class TestNonUniformCSV(TestCase): f.write(",".join(row) + "\n") def test_sheet_file_reader(self): - r = CSVFileReader(NamedContent(self.file_type, self.test_file)) + r = EncapsulatedSheetReader( + CSVFileReader(NamedContent(self.file_type, self.test_file)) + ) result = list(r.to_array()) self.assertEqual(result, [[1], [4, 5, 6], ["", 7]]) @@ -120,7 +127,9 @@ class TestNonUniformCSV(TestCase): def test_utf16_decoding(): test_file = os.path.join("tests", "fixtures", "csv-encoding-utf16.csv") - reader = CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") + reader = EncapsulatedSheetReader( + CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") + ) content = list(reader.to_array()) if PY2: content[0] = [s.encode("utf-8") for s in content[0]] @@ -147,8 +156,8 @@ def test_utf16_encoding(): def test_utf16_memory_decoding(): test_content = u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot" test_content = BytesIO(test_content.encode("utf-16")) - reader = CSVinMemoryReader( - NamedContent("csv", test_content), encoding="utf-16" + reader = EncapsulatedSheetReader( + CSVinMemoryReader(NamedContent("csv", test_content), encoding="utf-16") ) content = list(reader.to_array()) if PY2: diff --git a/tests/test_new_csv_book.py b/tests/test_new_csv_book.py index 930a58a..ab73b43 100644 --- a/tests/test_new_csv_book.py +++ b/tests/test_new_csv_book.py @@ -3,10 +3,9 @@ from textwrap import dedent from unittest import TestCase import pyexcel_io.manager as manager +from pyexcel_io.reader import Reader from pyexcel_io._compact import OrderedDict -from pyexcel_io.readers.tsv import TSVBookReader from pyexcel_io.writers.tsv import TSVBookWriter -from pyexcel_io.readers.csvr import CSVBookReader from pyexcel_io.writers.csvw import CSVBookWriter from nose.tools import raises @@ -14,9 +13,11 @@ from nose.tools import raises class TestCSVReaders(TestCase): file_type = "csv" - reader_class = CSVBookReader delimiter = "," + def reader_class(self): + return Reader(self.file_type) + def setUp(self): self.test_file = "csv_book." + self.file_type self.data = [["1", "2", "3"], ["4", "5", "6"], ["7", "8", "9"]] @@ -45,17 +46,50 @@ class TestCSVReaders(TestCase): os.unlink(self.test_file) -class TestTSVReaders(TestCSVReaders): +class TestNewCSVReaders(TestCase): + file_type = "csv" + delimiter = "," + + def setUp(self): + self.test_file = "csv_book." + self.file_type + self.data = [["1", "2", "3"], ["4", "5", "6"], ["7", "8", "9"]] + self.expected_data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] + with open(self.test_file, "w") as f: + for row in self.data: + f.write(self.delimiter.join(row) + "\n") + + def test_book_reader(self): + b = Reader(self.file_type) + b.open(self.test_file) + sheets = b.read_all() + self.assertEqual(list(sheets[self.test_file]), self.expected_data) + + def test_book_reader_from_memory_source(self): + io = manager.get_io(self.file_type) + with open(self.test_file, "r") as f: + io.write(f.read()) + io.seek(0) + b = Reader(self.file_type) + b.open_stream(io) + sheets = b.read_all() + self.assertEqual(list(sheets[self.file_type]), self.expected_data) + + def tearDown(self): + os.unlink(self.test_file) + + +class TestTSVReaders(TestNewCSVReaders): file_type = "tsv" - reader_class = TSVBookReader delimiter = "\t" class TestReadMultipleSheets(TestCase): file_type = "csv" - reader_class = CSVBookReader delimiter = "," + def reader_class(self): + return Reader(self.file_type) + def setUp(self): self.test_file_formatter = "csv_multiple__%s__%s." + self.file_type self.merged_book_file = "csv_multiple." + self.file_type @@ -123,14 +157,12 @@ class TestReadMultipleSheets(TestCase): class TestTSVBookReaders(TestReadMultipleSheets): file_type = "tsv" - reader_class = TSVBookReader delimiter = "\t" class TestWriteMultipleSheets(TestCase): file_type = "csv" writer_class = CSVBookWriter - reader_class = CSVBookReader result1 = dedent( """ 1,2,3 @@ -172,6 +204,9 @@ class TestWriteMultipleSheets(TestCase): """ ) + def reader_class(self): + return Reader(self.file_type) + def setUp(self): self.test_file_formatter = "csv_multiple__%s__%s." + self.file_type self.merged_book_file = "csv_multiple." + self.file_type @@ -242,7 +277,6 @@ class TestWriteMultipleSheets(TestCase): class TestTSVWriteMultipleSheets(TestWriteMultipleSheets): file_type = "tsv" writer_class = TSVBookWriter - reader_class = TSVBookReader result1 = dedent( """ 1\t2\t3 diff --git a/tests/test_new_csvz_book.py b/tests/test_new_csvz_book.py index b73faed..7850bbf 100644 --- a/tests/test_new_csvz_book.py +++ b/tests/test_new_csvz_book.py @@ -6,9 +6,8 @@ from unittest import TestCase import pyexcel_io.manager as manager from pyexcel_io import save_data +from pyexcel_io.reader import Reader from pyexcel_io._compact import OrderedDict -from pyexcel_io.readers.csvz import CSVZipBookReader -from pyexcel_io.readers.tsvz import TSVZipBookReader from pyexcel_io.writers.csvz import CSVZipBookWriter from pyexcel_io.writers.tsvz import TSVZipBookWriter @@ -20,9 +19,11 @@ PY2 = sys.version_info[0] == 2 class TestCSVZ(TestCase): file_type = "csvz" writer_class = CSVZipBookWriter - reader_class = CSVZipBookReader result = u"中,文,1,2,3" + def reader_class(self): + return Reader(self.file_type) + def setUp(self): self.file = "csvz." + self.file_type @@ -59,7 +60,6 @@ class TestCSVZ(TestCase): class TestTSVZ(TestCSVZ): file_type = "tsvz" writer_class = TSVZipBookWriter - reader_class = TSVZipBookReader result = u"中\t文\t1\t2\t3" @@ -70,7 +70,7 @@ def test_reading_from_memory(): zipbook.open_stream(io) zipbook.write({None: data}) zipbook.close() - zipreader = CSVZipBookReader() + zipreader = Reader("csvz") zipreader.open_stream(io) data = zipreader.read_all() assert list(data["pyexcel_sheet1"]) == [[1, 2, 3]] @@ -83,7 +83,7 @@ def test_reading_from_memory_tsvz(): zipbook.open_stream(io) zipbook.write({None: data}) zipbook.close() - zipreader = TSVZipBookReader() + zipreader = Reader("tsvz") zipreader.open_stream(io) data = zipreader.read_all() assert list(data["pyexcel_sheet1"]) == [[1, 2, 3]] @@ -91,7 +91,9 @@ def test_reading_from_memory_tsvz(): class TestMultipleSheet(TestCase): file_name = "mybook.csvz" - reader_class = CSVZipBookReader + + def reader_class(self): + return Reader("csvz") def setUp(self): self.content = OrderedDict() @@ -140,4 +142,6 @@ class TestMultipleSheet(TestCase): class TestMultipleTSVSheet(TestMultipleSheet): file_name = "mybook.tsvz" - reader_class = TSVZipBookReader + + def reader_class(self): + return Reader("tsvz") From de6344e11e37b5d2430267d8ebfc6de3a6a5b4c4 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 23 Aug 2020 01:21:19 +0100 Subject: [PATCH 069/143] :rocket: github actions for moban and automatically extract contributors --- .github/PULL_REQUEST_TEMPLATE.md | 5 ++-- .github/workflows/moban-update.yml | 27 ++++++++++++++++++++ CONTRIBUTORS.rst | 10 ++++++++ LICENSE | 2 +- Makefile | 16 +++++------- README.rst | 41 +++++++++++++----------------- docs/source/conf.py | 4 +-- setup.py | 2 +- 8 files changed, 67 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/moban-update.yml create mode 100644 CONTRIBUTORS.rst diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d5a2c03..6524ae3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,10 +3,9 @@ With your PR, here is a check list: - [ ] Has Test cases written - [ ] Has all code lines tested - [ ] Has `make format` been run? -- [ ] Has `moban` been run? +- [ ] Please update CHANGELOG.yml(not CHANGELOG.rst) - [ ] Passes all Travis CI builds - [ ] Has fair amount of documentation if your change is complex - [ ] run 'make format' so as to confirm the pyexcel organisation's coding style -- [ ] Please update CHANGELOG.rst -- [ ] Please add yourself to CONTRIBUTORS.rst +- [ ] Please add yourself to 'contributors' section of pyexcel-io.yml (if not found, please use CONTRIBUTORS.rst) - [ ] Agree on NEW BSD License for your contribution diff --git a/.github/workflows/moban-update.yml b/.github/workflows/moban-update.yml new file mode 100644 index 0000000..db4f13a --- /dev/null +++ b/.github/workflows/moban-update.yml @@ -0,0 +1,27 @@ +on: [push] + +jobs: + run_moban: + runs-on: ubuntu-latest + name: synchronize templates via moban + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.7' + - name: check changes + run: | + pip install moban gitfs2 pypifs + moban + git status + git diff --exit-code + - name: Auto-commit + if: failure() + uses: docker://cdssnc/auto-commit-github-action + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + args: This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst new file mode 100644 index 0000000..0ece1cb --- /dev/null +++ b/CONTRIBUTORS.rst @@ -0,0 +1,10 @@ + +4 contributors +================================================================================ + +In alphabetical order: + +* `Antherkiv `_ +* `John Vandenberg `_ +* `Stephen J. Fuhry `_ +* `Stephen Rauch `_ diff --git a/LICENSE b/LICENSE index e763169..802c39c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2019 by Onni Software Ltd. and its contributors +Copyright (c) 2015-2020 by Onni Software Ltd. and its contributors All rights reserved. Redistribution and use in source and binary forms of the software as well diff --git a/Makefile b/Makefile index 23f6e23..d7e640e 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,19 @@ all: test -test: +test: lint bash test.sh install_test: pip install -r tests/requirements.txt -document: - sphinx-autogen -o docs/source/generated/ docs/source/*.rst - sphinx-build -b html docs/source/ docs/build/ - -format: - isort -y $(find pyexcel_io -name "*.py"|xargs echo) $(find tests -name "*.py"|xargs echo) - black -l 79 pyexcel_io - black -l 79 tests +git-diff-check: + git diff --exit-code lint: bash lint.sh +format: + bash format.sh + git-diff-check: git diff --exit-code - diff --git a/README.rst b/README.rst index 0db73a2..984a923 100644 --- a/README.rst +++ b/README.rst @@ -98,6 +98,24 @@ sqlalchemy supported databases. Its supported file formats are extended to cover ======================== ======================= ================= ================== +Plugin shopping guide +------------------------ + +Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of +xml files + +The dedicated readers for excel files can stream read + + +In order to manage the list of plugins installed, you need to use pip to add or remove +a plugin. When you use virtualenv, you can have different plugins per virtual +environment. In the situation where you have multiple plugins that does the same thing +in your environment, you need to tell pyexcel which plugin to use per function call. +For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr. +You need to append get_array(..., library='pyexcel-odsr'). + + + .. _pyexcel-io: https://github.com/pyexcel/pyexcel-io .. _pyexcel-xls: https://github.com/pyexcel/pyexcel-xls .. _pyexcel-xlsx: https://github.com/pyexcel/pyexcel-xlsx @@ -120,13 +138,6 @@ sqlalchemy supported databases. Its supported file formats are extended to cover .. _odfpy: https://github.com/eea/odfpy -In order to manage the list of plugins installed, you need to use pip to add or remove -a plugin. When you use virtualenv, you can have different plugins per virtual -environment. In the situation where you have multiple plugins that does the same thing -in your environment, you need to tell pyexcel which plugin to use per function call. -For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr. -You need to append get_array(..., library='pyexcel-odsr'). - .. rubric:: Footnotes .. [#f1] zipped csv file @@ -214,22 +225,6 @@ Please run:: so as to beautify your code otherwise travis-ci may fail your unit test. -And make sure you would have run moban command ---------------------------------------------------------- - -Additional steps are required: - -#. pip install moban -#. make your changes in `.moban.d` directory, then issue command `moban` -#. moban - -otherwise travis-ci may also fail your unit test. - -What is .moban.d ---------------------------------- - -`.moban.d` stores the specific meta data for the library. - License diff --git a/docs/source/conf.py b/docs/source/conf.py index 29f8c7d..fa602ac 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,8 +23,8 @@ DESCRIPTION = ( # -- Project information ----------------------------------------------------- project = 'pyexcel-io' -copyright = '2015-2019 Onni Software Ltd.' -author = 'C.W.' +copyright = '2015-2020 Onni Software Ltd.' +author = 'chfw' # The short X.Y version version = '0.6.0' # The full version, including alpha/beta/rc tags diff --git a/setup.py b/setup.py index b955a71..19ec082 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ except (ValueError, UnicodeError, locale.Error): locale.setlocale(locale.LC_ALL, "en_US.UTF-8") NAME = "pyexcel-io" -AUTHOR = "C.W." +AUTHOR = "chfw" VERSION = "0.6.0" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" From a96bdc32c6c76a86ff81cbd1c713ced567efcb03 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 23 Aug 2020 00:21:58 +0000 Subject: [PATCH 070/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- format.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 format.sh diff --git a/format.sh b/format.sh new file mode 100644 index 0000000..73a38db --- /dev/null +++ b/format.sh @@ -0,0 +1,3 @@ +isort $(find pyexcel_io -name "*.py"|xargs echo) $(find tests -name "*.py"|xargs echo) +black -l 79 pyexcel_io +black -l 79 tests From 1ee31b0308a165230538b933ad22d4a045aa21f9 Mon Sep 17 00:00:00 2001 From: Craig Anderson Date: Sat, 19 Sep 2020 16:23:33 +0100 Subject: [PATCH 071/143] handle zip files which contain non-UTF-8 encoded files (#75) --- CHANGELOG.rst | 4 ++++ CONTRIBUTORS.rst | 1 + pyexcel_io/readers/csvz.py | 5 ++++- requirements.txt | 1 + setup.py | 1 + test.sh | 0 tests/test_new_csvz_book.py | 10 ++++++++++ 7 files changed, 21 insertions(+), 1 deletion(-) mode change 100644 => 100755 test.sh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9f22ef2..645e25d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ Change log 0.6.0 - tbd -------------------------------------------------------------------------------- +#. `#74 `_: handle zip files which + contain non-UTF-8 encoded files. + + **removed** #. python 3.6 lower versions are no longer supported diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 0ece1cb..9c17a94 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -5,6 +5,7 @@ In alphabetical order: * `Antherkiv `_ +* `Craig Anderson `_ * `John Vandenberg `_ * `Stephen J. Fuhry `_ * `Stephen Rauch `_ diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index 829a733..0844db1 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -9,6 +9,8 @@ """ import zipfile +import chardet + from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import StringIO from pyexcel_io.readers.csvr import CSVinMemoryReader @@ -43,7 +45,8 @@ class FileReader(object): def read_sheet(self, index): name = self.content_array[index].name content = self.zipfile.read(self.content_array[index].payload) - sheet = StringIO(content.decode("utf-8")) + encoding_guess = chardet.detect(content) + sheet = StringIO(content.decode(encoding_guess["encoding"])) return CSVinMemoryReader(NamedContent(name, sheet), **self.keywords) diff --git a/requirements.txt b/requirements.txt index dac001a..e8fc171 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ ordereddict;python_version<"2.7" lml>=0.0.4 +chardet diff --git a/setup.py b/setup.py index 19ec082..ee23512 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ PYTHON_REQUIRES = ">=3.6" INSTALL_REQUIRES = [ "lml>=0.0.4", + "chardet", ] SETUP_COMMANDS = {} diff --git a/test.sh b/test.sh old mode 100644 new mode 100755 diff --git a/tests/test_new_csvz_book.py b/tests/test_new_csvz_book.py index 7850bbf..5b4234a 100644 --- a/tests/test_new_csvz_book.py +++ b/tests/test_new_csvz_book.py @@ -53,6 +53,16 @@ class TestCSVZ(TestCase): self.assertEqual(list(data["pyexcel_sheet1"]), [[u"中", u"文", 1, 2, 3]]) zipreader.close() + def test_reading_utf32(self): + zip = zipfile.ZipFile(self.file, "w") + zip.writestr("something.ext", self.result.encode("utf-32")) + zip.close() + zipreader = self.reader_class() + zipreader.open(self.file) + data = zipreader.read_all() + self.assertEqual(list(data["something"]), [[u"中", u"文", 1, 2, 3]]) + zipreader.close() + def tearDown(self): os.unlink(self.file) From e205c503248df95f7fe3d67e5a195c6d6ef86721 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 19 Sep 2020 16:28:11 +0100 Subject: [PATCH 073/143] :books: relocate change log to changelog.yml, which will be rendered into CHANGELOG.rst. #75 --- changelog.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelog.yml b/changelog.yml index 395bb84..5295fb1 100644 --- a/changelog.yml +++ b/changelog.yml @@ -2,6 +2,9 @@ name: pyexcel-io organisation: pyexcel releases: - changes: + - action: fixed + details: + - "`#74`: handle zip files which contain non-UTF-8 encoded files." - action: removed details: - 'python 3.6 lower versions are no longer supported' From e8f798b4c82bb729018bd2fdc48f04e35845731d Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 19 Sep 2020 16:30:10 +0100 Subject: [PATCH 075/143] :green_heart: update moban update dependencies --- .github/workflows/moban-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/moban-update.yml b/.github/workflows/moban-update.yml index db4f13a..874ff61 100644 --- a/.github/workflows/moban-update.yml +++ b/.github/workflows/moban-update.yml @@ -14,7 +14,7 @@ jobs: python-version: '3.7' - name: check changes run: | - pip install moban gitfs2 pypifs + pip install moban gitfs2 pypifs moban-jinja2-github moban-ansible moban git status git diff --exit-code From 5a22a855acfb0101c68ce2e6cfabc6f15134d778 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 19 Sep 2020 16:36:41 +0100 Subject: [PATCH 076/143] :books: auto update(this is auto scraped from github profile) contributors and change log. #75 --- .github/workflows/moban-update.yml | 2 +- CHANGELOG.rst | 5 +++-- CONTRIBUTORS.rst | 4 ++-- requirements.txt | 1 - setup.py | 1 - test.sh | 0 tests/requirements.txt | 1 + 7 files changed, 7 insertions(+), 7 deletions(-) mode change 100755 => 100644 test.sh diff --git a/.github/workflows/moban-update.yml b/.github/workflows/moban-update.yml index 874ff61..db4f13a 100644 --- a/.github/workflows/moban-update.yml +++ b/.github/workflows/moban-update.yml @@ -14,7 +14,7 @@ jobs: python-version: '3.7' - name: check changes run: | - pip install moban gitfs2 pypifs moban-jinja2-github moban-ansible + pip install moban gitfs2 pypifs moban git status git diff --exit-code diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 645e25d..9e7e171 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,9 +4,10 @@ Change log 0.6.0 - tbd -------------------------------------------------------------------------------- -#. `#74 `_: handle zip files which - contain non-UTF-8 encoded files. +**fixed** +#. `#74 `_: handle zip files + which contain non-UTF-8 encoded files. **removed** diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 9c17a94..8f171f7 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -1,11 +1,11 @@ -4 contributors +5 contributors ================================================================================ In alphabetical order: -* `Antherkiv `_ * `Craig Anderson `_ * `John Vandenberg `_ * `Stephen J. Fuhry `_ * `Stephen Rauch `_ +* `Víctor Antonio Hernández Monroy `_ diff --git a/requirements.txt b/requirements.txt index e8fc171..dac001a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ ordereddict;python_version<"2.7" lml>=0.0.4 -chardet diff --git a/setup.py b/setup.py index ee23512..19ec082 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,6 @@ PYTHON_REQUIRES = ">=3.6" INSTALL_REQUIRES = [ "lml>=0.0.4", - "chardet", ] SETUP_COMMANDS = {} diff --git a/test.sh b/test.sh old mode 100755 new mode 100644 diff --git a/tests/requirements.txt b/tests/requirements.txt index ad5e0e3..2359d93 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -8,6 +8,7 @@ isort collective.checkdocs pygments moban +moban_jinja2_github pyexcel pyexcel-xls SQLAlchemy From 95842cf9acdc3a9ececcce810f6bbef4d057179d Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 19 Sep 2020 16:39:17 +0100 Subject: [PATCH 078/143] :lipstick: update coding style --- pyexcel_io/_compact.py | 2 +- pyexcel_io/database/exporters/django.py | 3 +-- pyexcel_io/database/exporters/sqlalchemy.py | 3 +-- pyexcel_io/database/importers/sqlalchemy.py | 3 +-- pyexcel_io/readers/csvz.py | 1 - pyexcel_io/readers/tsvz.py | 2 +- pyexcel_io/sheet.py | 3 +-- pyexcel_io/writers/tsvz.py | 2 +- tests/test_sql_book.py | 24 ++++++++++----------- 9 files changed, 19 insertions(+), 24 deletions(-) diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index 4e58bc3..c179ce7 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -50,7 +50,7 @@ if PY2: else: - from io import StringIO, BytesIO + from io import BytesIO, StringIO text_type = str Iterator = object diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index d871a07..862c1e1 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -12,8 +12,7 @@ from pyexcel_io.database.querysets import QuerysetsReader class DjangoModelReader(QuerysetsReader): - """Read from django model - """ + """Read from django model""" def __init__(self, model, export_columns=None, **keywords): self.__model = model diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index dffebdd..2ab2c8a 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -12,8 +12,7 @@ from pyexcel_io.database.querysets import QuerysetsReader class SQLTableReader(QuerysetsReader): - """Read a table - """ + """Read a table""" def __init__(self, session, table, export_columns=None, **keywords): everything = session.query(table).all() diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 597db91..1e4c3c2 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -23,8 +23,7 @@ class PyexcelSQLSkipRowException(Exception): class SQLTableWriter(SheetWriter): - """Write to a table - """ + """Write to a table""" def __init__( self, importer, adapter, auto_commit=True, bulk_size=1000, **keywords diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index 0844db1..b1ca68e 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -10,7 +10,6 @@ import zipfile import chardet - from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import StringIO from pyexcel_io.readers.csvr import CSVinMemoryReader diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py index 3ca5e91..1be5cba 100644 --- a/pyexcel_io/readers/tsvz.py +++ b/pyexcel_io/readers/tsvz.py @@ -13,7 +13,7 @@ from .csvz import FileReader class TSVZipFileReader(FileReader): - """ read zipped tab separated value file + """read zipped tab separated value file it supports single tsv file and mulitple tsv files """ diff --git a/pyexcel_io/sheet.py b/pyexcel_io/sheet.py index 7d54b57..5afd982 100644 --- a/pyexcel_io/sheet.py +++ b/pyexcel_io/sheet.py @@ -58,8 +58,7 @@ class SheetReader(object): self._skip_column = skip_column_func def to_array(self): - """2 dimentional representation of the content - """ + """2 dimentional representation of the content""" for row_index, row in enumerate(self.row_iterator()): row_position = self._skip_row( row_index, self._start_row, self._row_limit diff --git a/pyexcel_io/writers/tsvz.py b/pyexcel_io/writers/tsvz.py index 1528e4f..b4943ec 100644 --- a/pyexcel_io/writers/tsvz.py +++ b/pyexcel_io/writers/tsvz.py @@ -13,7 +13,7 @@ from .csvz import CSVZipBookWriter class TSVZipBookWriter(CSVZipBookWriter): - """ write zipped tsv file + """write zipped tsv file It is similiar to CSVZipBookWriter, but support tab separated values """ diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 47bc5c9..cba8177 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -3,6 +3,17 @@ import json import datetime import platform +from sqlalchemy import ( + Date, + Float, + Column, + String, + Integer, + DateTime, + ForeignKey, + create_engine, +) +from sqlalchemy.orm import backref, relationship, sessionmaker from pyexcel_io._compact import OrderedDict from pyexcel_io.database.common import ( SQLTableExporter, @@ -10,6 +21,7 @@ from pyexcel_io.database.common import ( SQLTableExportAdapter, SQLTableImportAdapter, ) +from sqlalchemy.ext.declarative import declarative_base from pyexcel_io.database.querysets import QuerysetsReader from pyexcel_io.database.exporters.sqlalchemy import ( SQLBookReader, @@ -22,18 +34,6 @@ from pyexcel_io.database.importers.sqlalchemy import ( ) from nose.tools import eq_, raises -from sqlalchemy import ( - Date, - Float, - Column, - String, - Integer, - DateTime, - ForeignKey, - create_engine, -) -from sqlalchemy.orm import backref, relationship, sessionmaker -from sqlalchemy.ext.declarative import declarative_base PY3 = sys.version_info[0] == 3 PY36 = PY3 and sys.version_info[1] == 6 From 4bc78d671c094773caa56b526a71575fe3e18883 Mon Sep 17 00:00:00 2001 From: jaska Date: Mon, 21 Sep 2020 22:17:07 +0100 Subject: [PATCH 080/143] New style writer (#76) * :hammer: code up the new writer interface * :horse: prototype * :tractor: new style csv writer for file, content and stream * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :green_heart: add test dependency * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :green_heart: update moban update yaml * :lipstick: update coding style with latest black, isort * :sparkles: tsv writers in new style * :horse: new style csvz writer * :horse: new style tsvz writer * :fire: remove old style csvz and tsvz writers * :fire: remove old style csv and tsv writer * :books: update changelog * :tractor: code refactoring and update :books: * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :tractor: rename sheet writer files * :fire: remove .moban.hashes * :fire: remove useless set_type function call * :lipstick: update coding style Co-authored-by: chfw --- .github/workflows/moban-update.yml | 42 ++++++------ CHANGELOG.rst | 5 ++ changelog.yml | 3 + docs/source/pyinstaller.rst | 30 ++++++++- pyexcel-io.yml | 1 + pyexcel_io/io.py | 8 ++- pyexcel_io/plugins.py | 47 ++++++++++++-- pyexcel_io/readers/__init__.py | 2 +- pyexcel_io/writer.py | 41 ++++++++---- pyexcel_io/writers/__init__.py | 26 ++++++-- pyexcel_io/writers/csv_in_file.py | 25 +++++++ pyexcel_io/writers/csv_in_memory.py | 24 +++++++ pyexcel_io/writers/{csvw.py => csv_sheet.py} | 26 -------- pyexcel_io/writers/csvz.py | 68 -------------------- pyexcel_io/writers/csvz_sheet.py | 32 +++++++++ pyexcel_io/writers/csvz_writer.py | 36 +++++++++++ pyexcel_io/writers/tsv.py | 24 ------- pyexcel_io/writers/tsv_in_file.py | 9 +++ pyexcel_io/writers/tsv_in_memory.py | 9 +++ pyexcel_io/writers/tsvz.py | 27 -------- pyexcel_io/writers/tsvz_writer.py | 11 ++++ tests/requirements.txt | 1 + tests/test_csv_book.py | 5 +- tests/test_new_csv_book.py | 30 +++++---- tests/test_new_csvz_book.py | 12 ++-- 25 files changed, 331 insertions(+), 213 deletions(-) create mode 100644 pyexcel_io/writers/csv_in_file.py create mode 100644 pyexcel_io/writers/csv_in_memory.py rename pyexcel_io/writers/{csvw.py => csv_sheet.py} (81%) delete mode 100644 pyexcel_io/writers/csvz.py create mode 100644 pyexcel_io/writers/csvz_sheet.py create mode 100644 pyexcel_io/writers/csvz_writer.py delete mode 100644 pyexcel_io/writers/tsv.py create mode 100644 pyexcel_io/writers/tsv_in_file.py create mode 100644 pyexcel_io/writers/tsv_in_memory.py delete mode 100644 pyexcel_io/writers/tsvz.py create mode 100644 pyexcel_io/writers/tsvz_writer.py diff --git a/.github/workflows/moban-update.yml b/.github/workflows/moban-update.yml index db4f13a..706fd82 100644 --- a/.github/workflows/moban-update.yml +++ b/.github/workflows/moban-update.yml @@ -5,23 +5,25 @@ jobs: runs-on: ubuntu-latest name: synchronize templates via moban steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.head_ref }} - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: '3.7' - - name: check changes - run: | - pip install moban gitfs2 pypifs - moban - git status - git diff --exit-code - - name: Auto-commit - if: failure() - uses: docker://cdssnc/auto-commit-github-action - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - args: This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst + - uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.7' + - name: check changes + run: | + pip install moban gitfs2 pypifs moban-jinja2-github moban-ansible + moban + git status + git diff --exit-code + - name: Auto-commit + if: failure() + uses: docker://cdssnc/auto-commit-github-action + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + args: >- + This is an auto-commit, updating project meta data, + such as changelog.rst, contributors.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9e7e171..fbc5526 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,11 @@ Change log #. python 3.6 lower versions are no longer supported +**updated** + +#. pyexcel-io plugin interface has been rewritten. PyInstaller user will be + impacted. + 0.5.20 - 17.7.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index 5295fb1..62a544f 100644 --- a/changelog.yml +++ b/changelog.yml @@ -8,6 +8,9 @@ releases: - action: removed details: - 'python 3.6 lower versions are no longer supported' + - action: updated + details: + - 'pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted.' version: 0.6.0 date: tbd - changes: diff --git a/docs/source/pyinstaller.rst b/docs/source/pyinstaller.rst index 531ccc7..d743033 100644 --- a/docs/source/pyinstaller.rst +++ b/docs/source/pyinstaller.rst @@ -1,6 +1,30 @@ Packaging with PyInstaller ================================================================================ +With pyexcel-io v0.6.0, the way to package it has been changed because +plugin interface update. + +Built-in plugins for pyexcel-io +--------------------------------- + +In order to package every built-in plugins of pyexcel-io, you need to specify:: + + --hidden-import pyexcel_io.readers.csvr + --hidden-import pyexcel_io.readers.csvz + --hidden-import pyexcel_io.readers.tsv + --hidden-import pyexcel_io.readers.tsvz + --hidden-import pyexcel_io.writers.csv_file_writer + --hidden-import pyexcel_io.writers.csv_memory_writer + --hidden-import pyexcel_io.writers.tsv_file_writer + --hidden-import pyexcel_io.writers.tsv_memory_writer + --hidden-import pyexcel_io.writers.csvz_writer + --hidden-import pyexcel_io.writers.tsvz_writer + --hidden-import pyexcel_io.database.importers.django + --hidden-import pyexcel_io.database.importers.sqlalchemy + --hidden-import pyexcel_io.database.exporters.django + --hidden-import pyexcel_io.database.exporters.sqlalchemy + + With pyexcel-io v0.4.0, the way to package it has been changed because it uses lml for all plugins. @@ -14,9 +38,9 @@ In order to package every built-in plugins of pyexcel-io, you need to specify:: --hidden-import pyexcel_io.readers.tsv --hidden-import pyexcel_io.readers.tsvz --hidden-import pyexcel_io.writers.csvw - --hidden-import pyexcel_io.readers.csvz - --hidden-import pyexcel_io.readers.tsv - --hidden-import pyexcel_io.readers.tsvz + --hidden-import pyexcel_io.writers.csvz + --hidden-import pyexcel_io.writers.tsv + --hidden-import pyexcel_io.writers.tsvz --hidden-import pyexcel_io.database.importers.django --hidden-import pyexcel_io.database.importers.sqlalchemy --hidden-import pyexcel_io.database.exporters.django diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 01e79e0..b7818fa 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -15,6 +15,7 @@ test_dependencies: - pyexcel - pyexcel-xls - SQLAlchemy + - pyexcel-xlsxw extra_dependencies: - xls: - pyexcel-xls>=0.5.0 diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index a02fc81..0674a63 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -14,7 +14,7 @@ from types import GeneratorType from pyexcel_io import constants from pyexcel_io.reader import Reader from pyexcel_io.writer import Writer -from pyexcel_io.plugins import OLD_READERS +from pyexcel_io.plugins import OLD_READERS, OLD_WRITERS from pyexcel_io._compact import isstream from pyexcel_io.exceptions import ( NoSupportingPluginFound, @@ -244,7 +244,11 @@ def get_writer( file_type_given = False - writer = Writer(file_type, library) + try: + writer = OLD_WRITERS.get_a_plugin(file_type, library) + except (NoSupportingPluginFound, SupportingPluginAvailableButNotInstalled): + writer = Writer(file_type, library) + if file_name: if file_type_given: writer.open_content(file_name, **keywords) diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index c403947..742a074 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -20,6 +20,7 @@ plugin compactibility table." READER_PLUGIN = "pyexcel-io reader" NEW_READER_PLUGIN = "pyexcel-io new reader" WRITER_PLUGIN = "pyexcel-io writer" +NEW_WRITER_PLUGIN = "pyexcel-io new writer" class IOPluginInfo(PluginInfo): @@ -83,6 +84,26 @@ class NewIOPluginInfoChain(PluginInfoChain): ) return self.add_a_plugin_instance(a_plugin_info) + def add_a_writer( + self, + relative_plugin_class_path=None, + locations=(), + file_types=(), + stream_type=None, + ): + """ add pyexcle-io writer plugin info """ + a_plugin_info = IOPluginInfo( + NEW_WRITER_PLUGIN, + self._get_abs_path(relative_plugin_class_path), + file_types=[ + f"{location}-{file_type}" + for file_type in file_types + for location in locations + ], + stream_type=stream_type, + ) + return self.add_a_plugin_instance(a_plugin_info) + class IOManager(PluginManager): """Manage pyexcel-io plugins""" @@ -143,12 +164,12 @@ class IOManager(PluginManager): class NewIOManager(IOManager): def load_me_later(self, plugin_info): PluginManager.load_me_later(self, plugin_info) - _do_additional_registration(plugin_info) + _do_additional_registration_for_new_plugins(plugin_info) def register_a_plugin(self, cls, plugin_info): """ for dynamically loaded plugin """ PluginManager.register_a_plugin(self, cls, plugin_info) - _do_additional_registration(plugin_info) + _do_additional_registration_for_new_plugins(plugin_info) def get_a_plugin( self, file_type=None, location=None, library=None, **keywords @@ -156,7 +177,6 @@ class NewIOManager(IOManager): __file_type = file_type.lower() plugin = self.load_me_now(f"{location}-{__file_type}", library=library) handler = plugin() - handler.set_type(__file_type) return handler def raise_exception(self, file_type): @@ -194,6 +214,16 @@ def _do_additional_registration(plugin_info): manager.register_a_file_type(file_type, plugin_info.stream_type, None) +def _do_additional_registration_for_new_plugins(plugin_info): + for file_type in plugin_info.tags(): + manager.register_stream_type( + file_type.split("-")[1], plugin_info.stream_type + ) + manager.register_a_file_type( + file_type.split("-")[1], plugin_info.stream_type, None + ) + + class FakeReaders: def get_all_formats(self): return OLD_READERS.get_all_formats().union( @@ -201,10 +231,19 @@ class FakeReaders: ) +class FakeWriters: + def get_all_formats(self): + return OLD_WRITERS.get_all_formats().union( + NEW_WRITERS.get_all_formats() + ) + + OLD_READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS) -WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS) +OLD_WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS) +NEW_WRITERS = NewIOManager(NEW_WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS) NEW_READERS = NewIOManager(NEW_READER_PLUGIN, ioutils.AVAILABLE_READERS) READERS = FakeReaders() +WRITERS = FakeWriters() def load_plugins(plugin_name_patterns, path, black_list, white_list): diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index c302e64..1d00577 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -7,7 +7,7 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.plugins import IOPluginInfoChain, NewIOPluginInfoChain +from pyexcel_io.plugins import NewIOPluginInfoChain NewIOPluginInfoChain(__name__).add_a_reader( relative_plugin_class_path="csv_file_reader.FileReader", diff --git a/pyexcel_io/writer.py b/pyexcel_io/writer.py index bdaf982..0e3e468 100644 --- a/pyexcel_io/writer.py +++ b/pyexcel_io/writer.py @@ -1,24 +1,43 @@ -from pyexcel_io.plugins import WRITERS +from pyexcel_io.plugins import NEW_WRITERS +from pyexcel_io._compact import isstream + +from .constants import MESSAGE_ERROR_03 class Writer(object): - def __init__(self, file_type, library): - self.writer = WRITERS.get_a_plugin(file_type, library) + def __init__(self, file_type, library=None): + self.file_type = file_type + self.library = library + self.keyboards = None def open(self, file_name, **keywords): + self.writer = NEW_WRITERS.get_a_plugin( + self.file_type, library=self.library, location="file" + ) self.writer.open(file_name, **keywords) - def open_stream(self, file_stream, **keywords): - self.writer.open_stream(file_stream, **keywords) - def open_content(self, file_stream, **keywords): - self.writer.open_content(file_stream, **keywords) + if not isstream(file_stream): + raise IOError(MESSAGE_ERROR_03) + self.writer = NEW_WRITERS.get_a_plugin( + self.file_type, library=self.library, location="content" + ) + self.writer.open(file_stream, **keywords) + + def open_stream(self, file_stream, **keywords): + self.writer = NEW_WRITERS.get_a_plugin( + self.file_type, library=self.library, location="memory" + ) + self.writer.open(file_stream, **keywords) def write(self, incoming_dict): - self.writer.write(incoming_dict) - - def create_sheet(self, sheet_name): - return self.writer.create_sheet(sheet_name) + for sheet_name in incoming_dict: + sheet_writer = self.writer.create_sheet(sheet_name) + if sheet_writer: + sheet_writer.write_array(incoming_dict[sheet_name]) + sheet_writer.close() + else: + raise Exception("Cannot create a sheet writer!") def close(self): self.writer.close() diff --git a/pyexcel_io/writers/__init__.py b/pyexcel_io/writers/__init__.py index f85263a..e76dc29 100644 --- a/pyexcel_io/writers/__init__.py +++ b/pyexcel_io/writers/__init__.py @@ -7,22 +7,36 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.plugins import IOPluginInfoChain +from pyexcel_io.plugins import NewIOPluginInfoChain -IOPluginInfoChain(__name__).add_a_writer( - relative_plugin_class_path="csvw.CSVBookWriter", +NewIOPluginInfoChain(__name__).add_a_writer( + relative_plugin_class_path="csv_in_file.CsvFileWriter", + locations=["file", "content"], file_types=["csv"], stream_type="text", ).add_a_writer( - relative_plugin_class_path="tsv.TSVBookWriter", + relative_plugin_class_path="csv_in_memory.CsvMemoryWriter", + locations=["memory"], + file_types=["csv"], + stream_type="text", +).add_a_writer( + relative_plugin_class_path="tsv_in_file.TsvFileWriter", + locations=["file", "content"], file_types=["tsv"], stream_type="text", ).add_a_writer( - relative_plugin_class_path="csvz.CSVZipBookWriter", + relative_plugin_class_path="tsv_in_memory.TsvMemoryWriter", + locations=["memory"], + file_types=["tsv"], + stream_type="text", +).add_a_writer( + relative_plugin_class_path="csvz_writer.CsvZipWriter", + locations=["memory", "file", "content"], file_types=["csvz"], stream_type="binary", ).add_a_writer( - relative_plugin_class_path="tsvz.TSVZipBookWriter", + relative_plugin_class_path="tsvz_writer.TsvZipWriter", + locations=["memory", "file", "content"], file_types=["tsvz"], stream_type="binary", ) diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py new file mode 100644 index 0000000..d4ddce0 --- /dev/null +++ b/pyexcel_io/writers/csv_in_file.py @@ -0,0 +1,25 @@ +from pyexcel_io.writers.csv_sheet import CSVFileWriter + + +class CsvFileWriter: + def __init__(self): + self.__index = 0 + self.writer = None + + def open(self, file_alike_object, **keywords): + self._file_alike_object = file_alike_object + self._keywords = keywords + + def create_sheet(self, name): + self.writer = CSVFileWriter( + self._file_alike_object, + name, + sheet_index=self.__index, + **self._keywords + ) + self.__index = self.__index + 1 + return self.writer + + def close(self): + if self.writer: + self.writer.close() diff --git a/pyexcel_io/writers/csv_in_memory.py b/pyexcel_io/writers/csv_in_memory.py new file mode 100644 index 0000000..3724439 --- /dev/null +++ b/pyexcel_io/writers/csv_in_memory.py @@ -0,0 +1,24 @@ +from pyexcel_io.writers.csv_sheet import CSVMemoryWriter + + +class CsvMemoryWriter: + def __init__(self): + self.__index = 0 + + def open(self, file_alike_object, **keywords): + self._file_alike_object = file_alike_object + self._keywords = keywords + + def create_sheet(self, name): + writer_class = CSVMemoryWriter + writer = writer_class( + self._file_alike_object, + name, + sheet_index=self.__index, + **self._keywords + ) + self.__index = self.__index + 1 + return writer + + def close(self): + pass diff --git a/pyexcel_io/writers/csvw.py b/pyexcel_io/writers/csv_sheet.py similarity index 81% rename from pyexcel_io/writers/csvw.py rename to pyexcel_io/writers/csv_sheet.py index e7036e0..16c1399 100644 --- a/pyexcel_io/writers/csvw.py +++ b/pyexcel_io/writers/csv_sheet.py @@ -9,9 +9,7 @@ """ import csv -import pyexcel_io._compact as compact import pyexcel_io.constants as constants -from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetWriter @@ -119,27 +117,3 @@ class CSVMemoryWriter(CSVSheetWriter): pass else: self.writer.writerow([constants.SEPARATOR_FORMATTER % ""]) - - -class CSVBookWriter(BookWriter): - """ write csv with unicode support """ - - def __init__(self): - BookWriter.__init__(self) - self._file_type = constants.FILE_FORMAT_CSV - self.__index = 0 - - def create_sheet(self, name): - writer_class = None - if compact.is_string(type(self._file_alike_object)): - writer_class = CSVFileWriter - else: - writer_class = CSVMemoryWriter - writer = writer_class( - self._file_alike_object, - name, - sheet_index=self.__index, - **self._keywords - ) - self.__index = self.__index + 1 - return writer diff --git a/pyexcel_io/writers/csvz.py b/pyexcel_io/writers/csvz.py deleted file mode 100644 index e4905d6..0000000 --- a/pyexcel_io/writers/csvz.py +++ /dev/null @@ -1,68 +0,0 @@ -""" - pyexcel_io.fileformat.csvz - ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - The lower level csvz file format handler. - - :copyright: (c) 2014-2020 by Onni Software Ltd. - :license: New BSD License, see LICENSE for more details -""" -import zipfile - -from pyexcel_io.book import BookWriter -from pyexcel_io._compact import StringIO -from pyexcel_io.constants import FILE_FORMAT_CSVZ, DEFAULT_SHEET_NAME - -from .csvw import CSVSheetWriter - - -class CSVZipSheetWriter(CSVSheetWriter): - """ handle the zipfile interface """ - - def __init__(self, zipfile, sheetname, file_extension, **keywords): - self.file_extension = file_extension - keywords["single_sheet_in_book"] = False - CSVSheetWriter.__init__(self, zipfile, sheetname, **keywords) - - def set_sheet_name(self, name): - self.content = StringIO() - import csv - - self.writer = csv.writer(self.content, **self._keywords) - - def close(self): - file_name = "%s.%s" % (self._native_sheet, self.file_extension) - self.content.seek(0) - self._native_book.writestr(file_name, self.content.read()) - self.content.close() - - -class CSVZipBookWriter(BookWriter): - """ - csvz writer - - It is better to store csv files as a csvz as it saves your disk space. - Pyexcel-io had the facility to unzip it for you or you could use - any other unzip software. - """ - - def __init__(self): - BookWriter.__init__(self) - self._file_type = FILE_FORMAT_CSVZ - self.zipfile = None - - def open(self, file_name, **keywords): - BookWriter.open(self, file_name, **keywords) - self.zipfile = zipfile.ZipFile(file_name, "w", zipfile.ZIP_DEFLATED) - - def create_sheet(self, name): - given_name = name - if given_name is None: - given_name = DEFAULT_SHEET_NAME - writer = CSVZipSheetWriter( - self.zipfile, given_name, self._file_type[:3], **self._keywords - ) - return writer - - def close(self): - self.zipfile.close() diff --git a/pyexcel_io/writers/csvz_sheet.py b/pyexcel_io/writers/csvz_sheet.py new file mode 100644 index 0000000..f4b0ee9 --- /dev/null +++ b/pyexcel_io/writers/csvz_sheet.py @@ -0,0 +1,32 @@ +""" + pyexcel_io.fileformat.csvz + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The lower level csvz file format handler. + + :copyright: (c) 2014-2020 by Onni Software Ltd. + :license: New BSD License, see LICENSE for more details +""" +from pyexcel_io._compact import StringIO +from pyexcel_io.writers.csv_sheet import CSVSheetWriter + + +class CSVZipSheetWriter(CSVSheetWriter): + """ handle the zipfile interface """ + + def __init__(self, zipfile, sheetname, file_extension, **keywords): + self.file_extension = file_extension + keywords["single_sheet_in_book"] = False + CSVSheetWriter.__init__(self, zipfile, sheetname, **keywords) + + def set_sheet_name(self, name): + self.content = StringIO() + import csv + + self.writer = csv.writer(self.content, **self._keywords) + + def close(self): + file_name = "%s.%s" % (self._native_sheet, self.file_extension) + self.content.seek(0) + self._native_book.writestr(file_name, self.content.read()) + self.content.close() diff --git a/pyexcel_io/writers/csvz_writer.py b/pyexcel_io/writers/csvz_writer.py new file mode 100644 index 0000000..da390fd --- /dev/null +++ b/pyexcel_io/writers/csvz_writer.py @@ -0,0 +1,36 @@ +import zipfile + +from pyexcel_io.constants import FILE_FORMAT_CSVZ, DEFAULT_SHEET_NAME +from pyexcel_io.writers.csvz_sheet import CSVZipSheetWriter + + +class CsvZipWriter(object): + """ + csvz writer + + It is better to store csv files as a csvz as it saves your disk space. + Pyexcel-io had the facility to unzip it for you or you could use + any other unzip software. + """ + + def __init__(self): + self.zipfile = None + self._keywords = None + self._file_type = FILE_FORMAT_CSVZ + + def open(self, file_name, **keywords): + self.zipfile = zipfile.ZipFile(file_name, "w", zipfile.ZIP_DEFLATED) + self._keywords = keywords + + def create_sheet(self, name): + given_name = name + if given_name is None: + given_name = DEFAULT_SHEET_NAME + writer = CSVZipSheetWriter( + self.zipfile, given_name, self._file_type[:3], **self._keywords + ) + return writer + + def close(self): + if self.zipfile: + self.zipfile.close() diff --git a/pyexcel_io/writers/tsv.py b/pyexcel_io/writers/tsv.py deleted file mode 100644 index 33f6577..0000000 --- a/pyexcel_io/writers/tsv.py +++ /dev/null @@ -1,24 +0,0 @@ -""" - pyexcel_io.fileformat.tsv - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - The lower level tsv file format handler. - - :copyright: (c) 2014-2020 by Onni Software Ltd. - :license: New BSD License, see LICENSE for more details -""" -import pyexcel_io.constants as constants - -from .csvw import CSVBookWriter - - -class TSVBookWriter(CSVBookWriter): - """ write tsv """ - - def __init__(self): - CSVBookWriter.__init__(self) - self._file_type = constants.FILE_FORMAT_TSV - - def open(self, file_name, **keywords): - keywords["dialect"] = constants.KEYWORD_TSV_DIALECT - CSVBookWriter.open(self, file_name, **keywords) diff --git a/pyexcel_io/writers/tsv_in_file.py b/pyexcel_io/writers/tsv_in_file.py new file mode 100644 index 0000000..03668ea --- /dev/null +++ b/pyexcel_io/writers/tsv_in_file.py @@ -0,0 +1,9 @@ +from pyexcel_io.constants import KEYWORD_TSV_DIALECT +from pyexcel_io.writers.csv_in_file import CsvFileWriter + + +class TsvFileWriter(CsvFileWriter): + def open(self, file_alike_object, **keywords): + super().open( + file_alike_object, dialect=KEYWORD_TSV_DIALECT, **keywords + ) diff --git a/pyexcel_io/writers/tsv_in_memory.py b/pyexcel_io/writers/tsv_in_memory.py new file mode 100644 index 0000000..e505c8b --- /dev/null +++ b/pyexcel_io/writers/tsv_in_memory.py @@ -0,0 +1,9 @@ +from pyexcel_io.constants import KEYWORD_TSV_DIALECT +from pyexcel_io.writers.csv_in_memory import CsvMemoryWriter + + +class TsvMemoryWriter(CsvMemoryWriter): + def open(self, file_alike_object, **keywords): + super().open( + file_alike_object, dialect=KEYWORD_TSV_DIALECT, **keywords + ) diff --git a/pyexcel_io/writers/tsvz.py b/pyexcel_io/writers/tsvz.py deleted file mode 100644 index b4943ec..0000000 --- a/pyexcel_io/writers/tsvz.py +++ /dev/null @@ -1,27 +0,0 @@ -""" - pyexcel_io.fileformat.tsvz - ~~~~~~~~~~~~~~~~~~~~~~~~~~ - - The lower level tsvz file format handler. - - :copyright: (c) 2014-2020 by Onni Software Ltd. - :license: New BSD License, see LICENSE for more details -""" -from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT - -from .csvz import CSVZipBookWriter - - -class TSVZipBookWriter(CSVZipBookWriter): - """write zipped tsv file - - It is similiar to CSVZipBookWriter, but support tab separated values - """ - - def __init__(self): - CSVZipBookWriter.__init__(self) - self._file_type = FILE_FORMAT_TSVZ - - def open(self, file_name, **keywords): - keywords["dialect"] = KEYWORD_TSV_DIALECT - CSVZipBookWriter.open(self, file_name, **keywords) diff --git a/pyexcel_io/writers/tsvz_writer.py b/pyexcel_io/writers/tsvz_writer.py new file mode 100644 index 0000000..09b83fe --- /dev/null +++ b/pyexcel_io/writers/tsvz_writer.py @@ -0,0 +1,11 @@ +from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT +from pyexcel_io.writers.csvz_writer import CsvZipWriter + + +class TsvZipWriter(CsvZipWriter): + def __init__(self): + super().__init__() + self._file_type = FILE_FORMAT_TSVZ + + def open(self, file_name, **keywords): + super().open(file_name, dialect=KEYWORD_TSV_DIALECT, **keywords) diff --git a/tests/requirements.txt b/tests/requirements.txt index 2359d93..f5ebd71 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -12,3 +12,4 @@ moban_jinja2_github pyexcel pyexcel-xls SQLAlchemy +pyexcel-xlsxw diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index 74a31c1..75175f8 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - import os from textwrap import dedent from unittest import TestCase @@ -14,7 +11,7 @@ from pyexcel_io.readers.csvr import ( CSVSheetReader, CSVinMemoryReader, ) -from pyexcel_io.writers.csvw import CSVFileWriter, CSVMemoryWriter +from pyexcel_io.writers.csv_sheet import CSVFileWriter, CSVMemoryWriter from nose.tools import eq_, raises diff --git a/tests/test_new_csv_book.py b/tests/test_new_csv_book.py index ab73b43..7655d40 100644 --- a/tests/test_new_csv_book.py +++ b/tests/test_new_csv_book.py @@ -4,9 +4,8 @@ from unittest import TestCase import pyexcel_io.manager as manager from pyexcel_io.reader import Reader +from pyexcel_io.writer import Writer from pyexcel_io._compact import OrderedDict -from pyexcel_io.writers.tsv import TSVBookWriter -from pyexcel_io.writers.csvw import CSVBookWriter from nose.tools import raises @@ -162,7 +161,7 @@ class TestTSVBookReaders(TestReadMultipleSheets): class TestWriteMultipleSheets(TestCase): file_type = "csv" - writer_class = CSVBookWriter + result1 = dedent( """ 1,2,3 @@ -204,6 +203,9 @@ class TestWriteMultipleSheets(TestCase): """ ) + def writer_class(self): + return Writer(self.file_type) + def reader_class(self): return Reader(self.file_type) @@ -246,7 +248,7 @@ class TestWriteMultipleSheets(TestCase): def test_multiple_sheet_into_memory(self): io = manager.get_io(self.file_type) w = self.writer_class() - w.open(io, lineterminator="\n") + w.open_stream(io, lineterminator="\n") w.write(self.sheets) w.close() content = io.getvalue() @@ -256,7 +258,7 @@ class TestWriteMultipleSheets(TestCase): """Write csv book into a single stream""" io = manager.get_io(self.file_type) w = self.writer_class() - w.open(io, lineterminator="\n") + w.open_stream(io, lineterminator="\n") w.write(self.sheets) w.close() reader = self.reader_class() @@ -276,7 +278,7 @@ class TestWriteMultipleSheets(TestCase): class TestTSVWriteMultipleSheets(TestWriteMultipleSheets): file_type = "tsv" - writer_class = TSVBookWriter + result1 = dedent( """ 1\t2\t3 @@ -321,7 +323,7 @@ class TestTSVWriteMultipleSheets(TestWriteMultipleSheets): class TestWriter(TestCase): file_type = "csv" - writer_class = CSVBookWriter + result = dedent( """ 1,2,3 @@ -330,6 +332,9 @@ class TestWriter(TestCase): """ ).strip("\n") + def writer_class(self): + return Writer(self.file_type) + def setUp(self): self.test_file = "csv_book." + self.file_type self.data = [["1", "2", "3"], ["4", "5", "6"], ["7", "8", "9"]] @@ -349,7 +354,7 @@ class TestWriter(TestCase): class TestTSVWriters(TestWriter): file_type = "tsv" - writer_class = TSVBookWriter + result = dedent( """ 1\t2\t3 @@ -361,7 +366,7 @@ class TestTSVWriters(TestWriter): class TestMemoryWriter(TestCase): file_type = "csv" - writer_class = CSVBookWriter + result = dedent( """ 1,2,3 @@ -370,6 +375,9 @@ class TestMemoryWriter(TestCase): """ ).strip("\n") + def writer_class(self): + return Writer(self.file_type) + def setUp(self): self.test_file = "csv_book." + self.file_type self.data = [["1", "2", "3"], ["4", "5", "6"], ["7", "8", "9"]] @@ -377,7 +385,7 @@ class TestMemoryWriter(TestCase): def test_book_writer_to_memroy(self): io = manager.get_io(self.file_type) w = self.writer_class() - w.open(io, single_sheet_in_book=True) + w.open_stream(io, single_sheet_in_book=True) w.write({self.file_type: self.data}) w.close() content = io.getvalue().replace("\r", "") @@ -386,7 +394,7 @@ class TestMemoryWriter(TestCase): class TestTSVMemoryWriter(TestMemoryWriter): file_type = "tsv" - writer_class = TSVBookWriter + result = dedent( """ 1\t2\t3 diff --git a/tests/test_new_csvz_book.py b/tests/test_new_csvz_book.py index 5b4234a..6f42b13 100644 --- a/tests/test_new_csvz_book.py +++ b/tests/test_new_csvz_book.py @@ -7,9 +7,8 @@ from unittest import TestCase import pyexcel_io.manager as manager from pyexcel_io import save_data from pyexcel_io.reader import Reader +from pyexcel_io.writer import Writer from pyexcel_io._compact import OrderedDict -from pyexcel_io.writers.csvz import CSVZipBookWriter -from pyexcel_io.writers.tsvz import TSVZipBookWriter from nose.tools import raises @@ -18,9 +17,11 @@ PY2 = sys.version_info[0] == 2 class TestCSVZ(TestCase): file_type = "csvz" - writer_class = CSVZipBookWriter result = u"中,文,1,2,3" + def writer_class(self): + return Writer(self.file_type) + def reader_class(self): return Reader(self.file_type) @@ -69,14 +70,13 @@ class TestCSVZ(TestCase): class TestTSVZ(TestCSVZ): file_type = "tsvz" - writer_class = TSVZipBookWriter result = u"中\t文\t1\t2\t3" def test_reading_from_memory(): data = [[1, 2, 3]] io = manager.get_io("csvz") - zipbook = CSVZipBookWriter() + zipbook = Writer("csvz") zipbook.open_stream(io) zipbook.write({None: data}) zipbook.close() @@ -89,7 +89,7 @@ def test_reading_from_memory(): def test_reading_from_memory_tsvz(): data = [[1, 2, 3]] io = manager.get_io("tsvz") - zipbook = TSVZipBookWriter() + zipbook = Writer("tsvz") zipbook.open_stream(io) zipbook.write({None: data}) zipbook.close() From d6f0f38896ae498800aa32357e84e41025462732 Mon Sep 17 00:00:00 2001 From: jaska Date: Tue, 22 Sep 2020 22:00:25 +0100 Subject: [PATCH 081/143] WIP : Code refactoring (#77) * :hammer: simplify the plugin interface. :fire: no open function, :fire: no more useless set_type, each plugin knows its type. * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :tractor: csvr to csv_sheet * :tractor: better naming for the readers * :handshake: update gitignore * :books: update pyinstaller notes * :fire: remmove python 2 related compactibility codes * :lipstick: update coding style * :fire: remove test files generated during tests Co-authored-by: chfw --- .gitignore | 552 +++++++++++++++++- .moban.yml | 1 + CONTRIBUTORS.rst | 10 +- MANIFEST.in | 5 +- docs/source/extendedcsv.rst | 2 + docs/source/pagination.rst | 6 + docs/source/plaincsv.rst | 1 + docs/source/pyinstaller.rst | 12 +- docs/source/renderer.rst | 7 + pyexcel_io/_compact.py | 33 +- pyexcel_io/plugins.py | 11 +- pyexcel_io/reader.py | 15 +- pyexcel_io/readers/__init__.py | 6 +- pyexcel_io/readers/csv_content.py | 30 + pyexcel_io/readers/csv_content_reader.py | 21 - .../{csv_file_reader.py => csv_in_file.py} | 11 +- ...{csv_memory_reader.py => csv_in_memory.py} | 12 +- pyexcel_io/readers/{csvr.py => csv_sheet.py} | 0 pyexcel_io/readers/csvz.py | 10 +- pyexcel_io/readers/tsv.py | 35 +- pyexcel_io/readers/tsvz.py | 5 +- pyexcel_io/writer.py | 12 +- pyexcel_io/writers/csv_in_file.py | 8 +- pyexcel_io/writers/csv_in_memory.py | 6 +- pyexcel_io/writers/csvz_writer.py | 6 +- pyexcel_io/writers/tsv_in_file.py | 4 +- pyexcel_io/writers/tsv_in_memory.py | 4 +- pyexcel_io/writers/tsvz_writer.py | 7 +- tests/test_book.py | 20 +- tests/test_csv_book.py | 12 +- tests/test_io.py | 3 + tests/test_issues.py | 55 +- tests/test_service.py | 26 - 33 files changed, 699 insertions(+), 249 deletions(-) create mode 100644 pyexcel_io/readers/csv_content.py delete mode 100644 pyexcel_io/readers/csv_content_reader.py rename pyexcel_io/readers/{csv_file_reader.py => csv_in_file.py} (91%) rename pyexcel_io/readers/{csv_memory_reader.py => csv_in_memory.py} (88%) rename pyexcel_io/readers/{csvr.py => csv_sheet.py} (100%) diff --git a/.gitignore b/.gitignore index 89f9f44..e8b12f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,546 @@ -*.pyc -*~ +# moban hashes +.moban.hashes + +# Extra rules from https://github.com/github/gitignore/ +# Python rules +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ .coverage -pyexcel*-info -build -dist -tmp.db -.idea/* +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# VirtualEnv rules +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +pip-selfcheck.json + +# Linux rules +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +# Windows rules +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# macOS rules +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Emacs rules +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + + +# Vim rules +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +# JetBrains rules +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +# SublimeText rules +# Cache files for Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# Workspace files are user-specific +*.sublime-workspace + +# Project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using Sublime Text +# *.sublime-project + +# SFTP configuration file +sftp-config.json +sftp-config-alt*.json + +# Package control specific files +Package Control.last-run +Package Control.ca-list +Package Control.ca-bundle +Package Control.system-ca-bundle +Package Control.cache/ +Package Control.ca-certs/ +Package Control.merged-ca-bundle +Package Control.user-ca-bundle +oscrypto-ca-bundle.crt +bh_unicode_properties.cache + +# Sublime-github package stores a github token in this file +# https://packagecontrol.io/packages/sublime-github +GitHub.sublime-settings + +# KDevelop4 rules +*.kdev4 +.kdev4/ + +# Kate rules +# Swap Files # +.*.kate-swp +.swp.* + +# TextMate rules +*.tmproj +*.tmproject +tmtags + +# VisualStudioCode rules +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Xcode rules +# Xcode +# +# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore + +## User settings +xcuserdata/ + +## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) +*.xcscmblueprint +*.xccheckout + +## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) +build/ +DerivedData/ +*.moved-aside +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 + +## Gcc Patch +/*.gcno + +# Eclipse rules +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +# TortoiseGit rules +# Project-level settings +/.tgitconfig + +# Tags rules +# Ignore tags created by etags, ctags, gtags (GNU global) and cscope +TAGS +.TAGS +!TAGS/ +tags +.tags +!tags/ +gtags.files +GTAGS +GRTAGS +GPATH +GSYMS +cscope.files +cscope.out +cscope.in.out +cscope.po.out + + +# remove moban hash dictionary +.moban.hashes diff --git a/.moban.yml b/.moban.yml index e6307f0..98dbc6f 100644 --- a/.moban.yml +++ b/.moban.yml @@ -7,3 +7,4 @@ targets: - .travis.yml: custom_travis.yml.jj2 - README.rst: io_readme.rst.jj2 - "docs/source/index.rst": "docs/source/index.rst" + - .gitignore: gitignore.jj2 diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 8f171f7..6d0fa16 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -4,8 +4,8 @@ In alphabetical order: -* `Craig Anderson `_ -* `John Vandenberg `_ -* `Stephen J. Fuhry `_ -* `Stephen Rauch `_ -* `Víctor Antonio Hernández Monroy `_ +* `Craig Anderson `_ +* `John Vandenberg `_ +* `Stephen J. Fuhry `_ +* `Stephen Rauch `_ +* `Víctor Antonio Hernández Monroy `_ diff --git a/MANIFEST.in b/MANIFEST.in index 6c71554..cadad1b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,8 @@ include README.rst include LICENSE include CHANGELOG.rst +include CONTRIBUTORS.rst recursive-include tests * -<<<<<<< HEAD -======= recursive-include docs * ->>>>>>> master -include docs/source/* include Makefile include test.sh diff --git a/docs/source/extendedcsv.rst b/docs/source/extendedcsv.rst index 64ba764..79318fd 100644 --- a/docs/source/extendedcsv.rst +++ b/docs/source/extendedcsv.rst @@ -74,6 +74,8 @@ Continue from previous example:: :hide: >>> import os + >>> if os.path.exists("your_file.csv"): + ... os.unlink("your_file.csv") >>> os.unlink("your_file__Sheet 1__0.csv") >>> os.unlink("your_file__Sheet 2__1.csv") diff --git a/docs/source/pagination.rst b/docs/source/pagination.rst index 51eeafe..cb9181a 100644 --- a/docs/source/pagination.rst +++ b/docs/source/pagination.rst @@ -60,3 +60,9 @@ Obvious, you could do both at the same time: The pagination support is available across all pyexcel-io plugins. +.. testcode:: + :hide: + + >>> import os + >>> if os.path.exists("your_file.csv"): + ... os.unlink("your_file.csv") diff --git a/docs/source/plaincsv.rst b/docs/source/plaincsv.rst index fe8f96c..c6c3490 100644 --- a/docs/source/plaincsv.rst +++ b/docs/source/plaincsv.rst @@ -172,4 +172,5 @@ When you read it back you will have to specify encoding too. >>> import os >>> os.unlink("your_file.csv") + >>> os.unlink("test-utf16-encoding.csv") >>> os.unlink(test_file) diff --git a/docs/source/pyinstaller.rst b/docs/source/pyinstaller.rst index d743033..d92179a 100644 --- a/docs/source/pyinstaller.rst +++ b/docs/source/pyinstaller.rst @@ -9,14 +9,16 @@ Built-in plugins for pyexcel-io In order to package every built-in plugins of pyexcel-io, you need to specify:: - --hidden-import pyexcel_io.readers.csvr + --hidden-import pyexcel_io.readers.csv_in_file + --hidden-import pyexcel_io.readers.csv_in_memory + --hidden-import pyexcel_io.readers.csv_content --hidden-import pyexcel_io.readers.csvz --hidden-import pyexcel_io.readers.tsv --hidden-import pyexcel_io.readers.tsvz - --hidden-import pyexcel_io.writers.csv_file_writer - --hidden-import pyexcel_io.writers.csv_memory_writer - --hidden-import pyexcel_io.writers.tsv_file_writer - --hidden-import pyexcel_io.writers.tsv_memory_writer + --hidden-import pyexcel_io.writers.csv_in_file + --hidden-import pyexcel_io.writers.csv_in_memory + --hidden-import pyexcel_io.writers.tsv_in_file + --hidden-import pyexcel_io.writers.tsv_in_memory --hidden-import pyexcel_io.writers.csvz_writer --hidden-import pyexcel_io.writers.tsvz_writer --hidden-import pyexcel_io.database.importers.django diff --git a/docs/source/renderer.rst b/docs/source/renderer.rst index a10631e..24a6f0a 100644 --- a/docs/source/renderer.rst +++ b/docs/source/renderer.rst @@ -46,3 +46,10 @@ And you may want use row_renderer to customize it to string: >>> data = get_data("your_file.csv", row_renderer=my_renderer) >>> data['your_file.csv'] [['1', '21', '31'], ['2', '22', '32'], ['3', '23', '33']] + +.. testcode:: + :hide: + + >>> import os + >>> if os.path.exists("your_file.csv"): + ... os.unlink("your_file.csv") diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index c179ce7..37a5694 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -16,17 +16,7 @@ import sys import types import logging - -PY2 = sys.version_info[0] == 2 -PY3_ABOVE = sys.version_info[0] >= 3 -PY26 = PY2 and sys.version_info[1] < 7 -PY27 = PY2 and sys.version_info[1] == 7 -PY27_ABOVE = PY27 or PY3_ABOVE - -if PY26: - from ordereddict import OrderedDict -else: - from collections import OrderedDict +from collections import OrderedDict try: from logging import NullHandler @@ -37,24 +27,11 @@ except ImportError: pass -if PY2: - from cStringIO import StringIO - from cStringIO import StringIO as BytesIO +from io import BytesIO, StringIO - text_type = unicode - irange = xrange - - class Iterator(object): - def next(self): - return type(self).__next__(self) - - -else: - from io import BytesIO, StringIO - - text_type = str - Iterator = object - irange = range +text_type = str +Iterator = object +irange = range def isstream(instance): diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 742a074..89b0e6e 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -176,8 +176,7 @@ class NewIOManager(IOManager): ): __file_type = file_type.lower() plugin = self.load_me_now(f"{location}-{__file_type}", library=library) - handler = plugin() - return handler + return plugin def raise_exception(self, file_type): file_type = file_type.split("-")[1] @@ -224,14 +223,14 @@ def _do_additional_registration_for_new_plugins(plugin_info): ) -class FakeReaders: +class AllReaders: def get_all_formats(self): return OLD_READERS.get_all_formats().union( NEW_READERS.get_all_formats() ) -class FakeWriters: +class AllWriters: def get_all_formats(self): return OLD_WRITERS.get_all_formats().union( NEW_WRITERS.get_all_formats() @@ -242,8 +241,8 @@ OLD_READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS) OLD_WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS) NEW_WRITERS = NewIOManager(NEW_WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS) NEW_READERS = NewIOManager(NEW_READER_PLUGIN, ioutils.AVAILABLE_READERS) -READERS = FakeReaders() -WRITERS = FakeWriters() +READERS = AllReaders() +WRITERS = AllWriters() def load_plugins(plugin_name_patterns, path, black_list, white_list): diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py index 1103938..0d9a9aa 100644 --- a/pyexcel_io/reader.py +++ b/pyexcel_io/reader.py @@ -33,19 +33,21 @@ class Reader(object): self.keywords = None def open(self, file_name, **keywords): - self.reader = NEW_READERS.get_a_plugin( + reader_class = NEW_READERS.get_a_plugin( self.file_type, location="file", library=self.library ) self.keywords, native_sheet_keywords = clean_keywords(keywords) - return self.reader.open(file_name, **native_sheet_keywords) + self.reader = reader_class(file_name, **native_sheet_keywords) + return self.reader def open_content(self, file_content, **keywords): self.keywords, native_sheet_keywords = clean_keywords(keywords) try: - self.reader = NEW_READERS.get_a_plugin( + reader_class = NEW_READERS.get_a_plugin( self.file_type, location="content", library=self.library ) - return self.reader.open(file_content, **native_sheet_keywords) + self.reader = reader_class(file_content, **native_sheet_keywords) + return self.reader except ( exceptions.NoSupportingPluginFound, exceptions.SupportingPluginAvailableButNotInstalled, @@ -57,10 +59,11 @@ class Reader(object): def open_stream(self, file_stream, **keywords): self.keywords, native_sheet_keywords = clean_keywords(keywords) - self.reader = NEW_READERS.get_a_plugin( + reader_class = NEW_READERS.get_a_plugin( self.file_type, location="memory", library=self.library ) - return self.reader.open(file_stream, **native_sheet_keywords) + self.reader = reader_class(file_stream, **native_sheet_keywords) + return self.reader def read_sheet_by_name(self, sheet_name): """ diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index 1d00577..9cb3cb2 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -10,17 +10,17 @@ from pyexcel_io.plugins import NewIOPluginInfoChain NewIOPluginInfoChain(__name__).add_a_reader( - relative_plugin_class_path="csv_file_reader.FileReader", + relative_plugin_class_path="csv_in_file.FileReader", location="file", file_types=["csv"], stream_type="text", ).add_a_reader( - relative_plugin_class_path="csv_content_reader.ContentReader", + relative_plugin_class_path="csv_content.ContentReader", location="content", file_types=["csv"], stream_type="text", ).add_a_reader( - relative_plugin_class_path="csv_memory_reader.MemoryReader", + relative_plugin_class_path="csv_in_memory.MemoryReader", location="memory", file_types=["csv"], stream_type="text", diff --git a/pyexcel_io/readers/csv_content.py b/pyexcel_io/readers/csv_content.py new file mode 100644 index 0000000..96636cf --- /dev/null +++ b/pyexcel_io/readers/csv_content.py @@ -0,0 +1,30 @@ +import mmap + +import pyexcel_io.constants as constants +from pyexcel_io.book import _convert_content_to_stream +from pyexcel_io.readers.csv_sheet import CSVMemoryMapIterator +from pyexcel_io.readers.csv_in_memory import MemoryReader + + +class ContentReader(MemoryReader): + file_type = constants.FILE_FORMAT_CSV + + def __init__(self, file_content, **keywords): + file_stream = ContentReader.convert_content_to_stream( + file_content, self.file_type, **keywords + ) + super().__init__(file_stream, **keywords) + + @staticmethod + def convert_content_to_stream(file_content, file_type, **keywords): + encoding = keywords.get("encoding", "utf-8") + if isinstance(file_content, mmap.mmap): + # load from mmap + file_stream = CSVMemoryMapIterator(file_content, encoding) + else: + if isinstance(file_content, bytes): + file_content = file_content.decode(encoding) + + file_stream = _convert_content_to_stream(file_content, file_type) + + return file_stream diff --git a/pyexcel_io/readers/csv_content_reader.py b/pyexcel_io/readers/csv_content_reader.py deleted file mode 100644 index 52f29dc..0000000 --- a/pyexcel_io/readers/csv_content_reader.py +++ /dev/null @@ -1,21 +0,0 @@ -import mmap - -from pyexcel_io.book import _convert_content_to_stream -from pyexcel_io.readers.csvr import CSVMemoryMapIterator -from pyexcel_io.readers.csv_memory_reader import MemoryReader - - -class ContentReader(MemoryReader): - def open(self, file_content, **keywords): - encoding = keywords.get("encoding", "utf-8") - if isinstance(file_content, mmap.mmap): - # load from mmap - file_stream = CSVMemoryMapIterator(file_content, encoding) - else: - if isinstance(file_content, bytes): - file_content = file_content.decode(encoding) - - file_stream = _convert_content_to_stream( - file_content, self.file_type - ) - super(ContentReader, self).open(file_stream, **keywords) diff --git a/pyexcel_io/readers/csv_file_reader.py b/pyexcel_io/readers/csv_in_file.py similarity index 91% rename from pyexcel_io/readers/csv_file_reader.py rename to pyexcel_io/readers/csv_in_file.py index 5f196f0..fa68618 100644 --- a/pyexcel_io/readers/csv_file_reader.py +++ b/pyexcel_io/readers/csv_in_file.py @@ -4,23 +4,18 @@ import glob from pyexcel_io import constants from pyexcel_io.sheet import NamedContent -from pyexcel_io.readers.csvr import CSVFileReader +from pyexcel_io.readers.csv_sheet import CSVFileReader DEFAULT_NEWLINE = "\r\n" class FileReader(object): - def __init__(self): - self.handles = [] - - def set_type(self, _): - pass - - def open(self, file_name, **keywords): + def __init__(self, file_name, **keywords): """Load content from a file :params str filename: an accessible file path :returns: a book """ + self.handles = [] self.keywords = keywords self.__line_terminator = keywords.get( constants.KEYWORD_LINE_TERMINATOR, DEFAULT_NEWLINE diff --git a/pyexcel_io/readers/csv_memory_reader.py b/pyexcel_io/readers/csv_in_memory.py similarity index 88% rename from pyexcel_io/readers/csv_memory_reader.py rename to pyexcel_io/readers/csv_in_memory.py index 830d0f2..aa54693 100644 --- a/pyexcel_io/readers/csv_memory_reader.py +++ b/pyexcel_io/readers/csv_in_memory.py @@ -3,24 +3,20 @@ import re import pyexcel_io._compact as compact from pyexcel_io import constants from pyexcel_io.sheet import NamedContent -from pyexcel_io.readers.csvr import CSVinMemoryReader +from pyexcel_io.readers.csv_sheet import CSVinMemoryReader DEFAULT_SHEET_SEPARATOR_FORMATTER = f"---{constants.DEFAULT_NAME}---%s" class MemoryReader(object): - def __init__(self): - self.handles = [] - self.file_type = constants.FILE_FORMAT_CSV + file_type = constants.FILE_FORMAT_CSV - def set_type(self, _): - pass - - def open(self, file_stream, multiple_sheets=False, **keywords): + def __init__(self, file_stream, multiple_sheets=False, **keywords): """Load content from memory :params stream file_content: the actual file content in memory :returns: a book """ + self.handles = [] self.keywords = keywords self.__load_from_memory_flag = True self.__line_terminator = keywords.get( diff --git a/pyexcel_io/readers/csvr.py b/pyexcel_io/readers/csv_sheet.py similarity index 100% rename from pyexcel_io/readers/csvr.py rename to pyexcel_io/readers/csv_sheet.py diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index b1ca68e..a783071 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -12,18 +12,12 @@ import zipfile import chardet from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import StringIO -from pyexcel_io.readers.csvr import CSVinMemoryReader +from pyexcel_io.readers.csv_sheet import CSVinMemoryReader class FileReader(object): - def __init__(self): + def __init__(self, file_alike_object, **keywords): self.content_array = [] - self.keywords = None - - def set_type(self, _): - pass - - def open(self, file_alike_object, **keywords): try: self.zipfile = zipfile.ZipFile(file_alike_object, "r") sheets = [ diff --git a/pyexcel_io/readers/tsv.py b/pyexcel_io/readers/tsv.py index d505994..79edea9 100644 --- a/pyexcel_io/readers/tsv.py +++ b/pyexcel_io/readers/tsv.py @@ -8,32 +8,31 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants -from pyexcel_io.readers.csv_file_reader import FileReader -from pyexcel_io.readers.csv_memory_reader import MemoryReader -from pyexcel_io.readers.csv_content_reader import ContentReader +from pyexcel_io.readers.csv_content import ContentReader +from pyexcel_io.readers.csv_in_file import FileReader +from pyexcel_io.readers.csv_in_memory import MemoryReader class TSVFileReader(FileReader): - def open(self, file_name, **keywords): - keywords["dialect"] = constants.KEYWORD_TSV_DIALECT - super(TSVFileReader, self).open(file_name, **keywords) + def __init__(self, file_name, **keywords): + super().__init__( + file_name, dialect=constants.KEYWORD_TSV_DIALECT, **keywords + ) class TSVMemoryReader(MemoryReader): - def __init__(self): - self.handles = [] - self.file_type = constants.FILE_FORMAT_TSV + file_type = constants.FILE_FORMAT_TSV - def open(self, file_stream, **keywords): - keywords["dialect"] = constants.KEYWORD_TSV_DIALECT - super(TSVMemoryReader, self).open(file_stream, **keywords) + def __init__(self, file_stream, **keywords): + super().__init__( + file_stream, dialect=constants.KEYWORD_TSV_DIALECT, **keywords + ) class TSVContentReader(ContentReader): - def __init__(self): - self.handles = [] - self.file_type = constants.FILE_FORMAT_TSV + file_type = constants.FILE_FORMAT_TSV - def open(self, file_content, **keywords): - keywords["dialect"] = constants.KEYWORD_TSV_DIALECT - super(TSVContentReader, self).open(file_content, **keywords) + def __init__(self, file_content, **keywords): + super().__init__( + file_content, dialect=constants.KEYWORD_TSV_DIALECT, **keywords + ) diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py index 1be5cba..bd6d904 100644 --- a/pyexcel_io/readers/tsvz.py +++ b/pyexcel_io/readers/tsvz.py @@ -18,6 +18,5 @@ class TSVZipFileReader(FileReader): it supports single tsv file and mulitple tsv files """ - def open(self, file_name, **keywords): - keywords["dialect"] = KEYWORD_TSV_DIALECT - super(TSVZipFileReader, self).open(file_name, **keywords) + def __init__(self, file_name, **keywords): + super().__init__(file_name, dialect=KEYWORD_TSV_DIALECT, **keywords) diff --git a/pyexcel_io/writer.py b/pyexcel_io/writer.py index 0e3e468..493f75b 100644 --- a/pyexcel_io/writer.py +++ b/pyexcel_io/writer.py @@ -11,24 +11,24 @@ class Writer(object): self.keyboards = None def open(self, file_name, **keywords): - self.writer = NEW_WRITERS.get_a_plugin( + writer_class = NEW_WRITERS.get_a_plugin( self.file_type, library=self.library, location="file" ) - self.writer.open(file_name, **keywords) + self.writer = writer_class(file_name, **keywords) def open_content(self, file_stream, **keywords): if not isstream(file_stream): raise IOError(MESSAGE_ERROR_03) - self.writer = NEW_WRITERS.get_a_plugin( + writer_class = NEW_WRITERS.get_a_plugin( self.file_type, library=self.library, location="content" ) - self.writer.open(file_stream, **keywords) + self.writer = writer_class(file_stream, **keywords) def open_stream(self, file_stream, **keywords): - self.writer = NEW_WRITERS.get_a_plugin( + writer_class = NEW_WRITERS.get_a_plugin( self.file_type, library=self.library, location="memory" ) - self.writer.open(file_stream, **keywords) + self.writer = writer_class(file_stream, **keywords) def write(self, incoming_dict): for sheet_name in incoming_dict: diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py index d4ddce0..f093cdf 100644 --- a/pyexcel_io/writers/csv_in_file.py +++ b/pyexcel_io/writers/csv_in_file.py @@ -2,13 +2,11 @@ from pyexcel_io.writers.csv_sheet import CSVFileWriter class CsvFileWriter: - def __init__(self): - self.__index = 0 - self.writer = None - - def open(self, file_alike_object, **keywords): + def __init__(self, file_alike_object, **keywords): self._file_alike_object = file_alike_object self._keywords = keywords + self.__index = 0 + self.writer = None def create_sheet(self, name): self.writer = CSVFileWriter( diff --git a/pyexcel_io/writers/csv_in_memory.py b/pyexcel_io/writers/csv_in_memory.py index 3724439..a3ad8b2 100644 --- a/pyexcel_io/writers/csv_in_memory.py +++ b/pyexcel_io/writers/csv_in_memory.py @@ -2,12 +2,10 @@ from pyexcel_io.writers.csv_sheet import CSVMemoryWriter class CsvMemoryWriter: - def __init__(self): - self.__index = 0 - - def open(self, file_alike_object, **keywords): + def __init__(self, file_alike_object, **keywords): self._file_alike_object = file_alike_object self._keywords = keywords + self.__index = 0 def create_sheet(self, name): writer_class = CSVMemoryWriter diff --git a/pyexcel_io/writers/csvz_writer.py b/pyexcel_io/writers/csvz_writer.py index da390fd..19bffff 100644 --- a/pyexcel_io/writers/csvz_writer.py +++ b/pyexcel_io/writers/csvz_writer.py @@ -13,12 +13,8 @@ class CsvZipWriter(object): any other unzip software. """ - def __init__(self): - self.zipfile = None - self._keywords = None + def __init__(self, file_name, **keywords): self._file_type = FILE_FORMAT_CSVZ - - def open(self, file_name, **keywords): self.zipfile = zipfile.ZipFile(file_name, "w", zipfile.ZIP_DEFLATED) self._keywords = keywords diff --git a/pyexcel_io/writers/tsv_in_file.py b/pyexcel_io/writers/tsv_in_file.py index 03668ea..853285d 100644 --- a/pyexcel_io/writers/tsv_in_file.py +++ b/pyexcel_io/writers/tsv_in_file.py @@ -3,7 +3,7 @@ from pyexcel_io.writers.csv_in_file import CsvFileWriter class TsvFileWriter(CsvFileWriter): - def open(self, file_alike_object, **keywords): - super().open( + def __init__(self, file_alike_object, **keywords): + super().__init__( file_alike_object, dialect=KEYWORD_TSV_DIALECT, **keywords ) diff --git a/pyexcel_io/writers/tsv_in_memory.py b/pyexcel_io/writers/tsv_in_memory.py index e505c8b..0e1d305 100644 --- a/pyexcel_io/writers/tsv_in_memory.py +++ b/pyexcel_io/writers/tsv_in_memory.py @@ -3,7 +3,7 @@ from pyexcel_io.writers.csv_in_memory import CsvMemoryWriter class TsvMemoryWriter(CsvMemoryWriter): - def open(self, file_alike_object, **keywords): - super().open( + def __init__(self, file_alike_object, **keywords): + super().__init__( file_alike_object, dialect=KEYWORD_TSV_DIALECT, **keywords ) diff --git a/pyexcel_io/writers/tsvz_writer.py b/pyexcel_io/writers/tsvz_writer.py index 09b83fe..7eb438d 100644 --- a/pyexcel_io/writers/tsvz_writer.py +++ b/pyexcel_io/writers/tsvz_writer.py @@ -3,9 +3,6 @@ from pyexcel_io.writers.csvz_writer import CsvZipWriter class TsvZipWriter(CsvZipWriter): - def __init__(self): - super().__init__() + def __init__(self, file_name, **keywords): + super().__init__(file_name, dialect=KEYWORD_TSV_DIALECT, **keywords) self._file_type = FILE_FORMAT_TSVZ - - def open(self, file_name, **keywords): - super().open(file_name, dialect=KEYWORD_TSV_DIALECT, **keywords) diff --git a/tests/test_book.py b/tests/test_book.py index 14755a2..c14a380 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -4,7 +4,7 @@ from pyexcel_io.book import ( RWInterface, _convert_content_to_stream, ) -from pyexcel_io._compact import PY2, BytesIO, StringIO +from pyexcel_io._compact import BytesIO, StringIO from nose import SkipTest from nose.tools import raises @@ -41,18 +41,12 @@ def test_book_writer(): def test_convert_to_bytes_stream(): - if PY2: - raise SkipTest("No need test in python 2") - else: - file_content = b"test" - stream = _convert_content_to_stream(file_content, "csv") - assert isinstance(stream, StringIO) + file_content = b"test" + stream = _convert_content_to_stream(file_content, "csv") + assert isinstance(stream, StringIO) def test_convert_to_string_stream(): - if PY2: - raise SkipTest("No need test in python 2") - else: - file_content = "test" - stream = _convert_content_to_stream(file_content, "csvz") - assert isinstance(stream, BytesIO) + file_content = "test" + stream = _convert_content_to_stream(file_content, "csvz") + assert isinstance(stream, BytesIO) diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index 75175f8..381ca69 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -5,8 +5,8 @@ from unittest import TestCase import pyexcel_io.manager as manager from pyexcel_io.sheet import NamedContent from pyexcel_io.reader import EncapsulatedSheetReader -from pyexcel_io._compact import PY2, BytesIO, StringIO -from pyexcel_io.readers.csvr import ( +from pyexcel_io._compact import BytesIO, StringIO +from pyexcel_io.readers.csv_sheet import ( CSVFileReader, CSVSheetReader, CSVinMemoryReader, @@ -128,8 +128,6 @@ def test_utf16_decoding(): CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") ) content = list(reader.to_array()) - if PY2: - content[0] = [s.encode("utf-8") for s in content[0]] expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] eq_(content, expected) @@ -144,8 +142,6 @@ def test_utf16_encoding(): writer.close() with open(test_file, "rb") as f: actual = f.read().decode("utf-16") - if PY2: - actual = actual.encode("utf-8") eq_(actual, "Äkkilähdöt,Matkakirjoituksia,Matkatoimistot\n") os.unlink(test_file) @@ -157,8 +153,6 @@ def test_utf16_memory_decoding(): CSVinMemoryReader(NamedContent("csv", test_content), encoding="utf-16") ) content = list(reader.to_array()) - if PY2: - content[0] = [s.encode("utf-8") for s in content[0]] expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] eq_(content, expected) @@ -175,6 +169,4 @@ def test_utf16_memory_encoding(): ) writer.write_array(content) actual = io.getvalue() - if PY2: - actual = actual.decode("utf-16") eq_(actual, u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot\n") diff --git a/tests/test_io.py b/tests/test_io.py index 1870c74..0646244 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -196,6 +196,7 @@ def test_file_handle_as_input(): with open(test_file, "r") as f: data = get_data(f, "csv") eq_(data["csv"], [[1, 2, 3]]) + os.unlink("file_handle.csv") def test_file_type_case_insensitivity(): @@ -206,6 +207,7 @@ def test_file_type_case_insensitivity(): with open(test_file, "r") as f: data = get_data(f, "csv") eq_(data["csv"], [[1, 2, 3]]) + os.unlink("file_handle.CSv") def test_file_handle_as_output(): @@ -216,6 +218,7 @@ def test_file_handle_as_output(): with open(test_file, "r") as f: content = f.read() eq_(content, "1,2,3\n") + os.unlink("file_handle.csv") def test_binary_file_content(): diff --git a/tests/test_issues.py b/tests/test_issues.py index 09582e0..b7b9dd5 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -5,7 +5,6 @@ import os import pyexcel as p from pyexcel_io import get_data, save_data -from pyexcel_io._compact import PY26 from nose import SkipTest from nose.tools import eq_ @@ -54,19 +53,14 @@ def test_issue_23(): def test_issue_33_34(): - if PY26: - pass - else: - import mmap + import mmap - test_file = get_fixture("issue20.csv") - with open(test_file, "r+b") as f: - memory_mapped_file = mmap.mmap( - f.fileno(), 0, access=mmap.ACCESS_READ - ) - data = get_data(memory_mapped_file, file_type="csv") - expected = [[u"to", u"infinity", u"and", u"beyond"]] - eq_(data["csv"], expected) + test_file = get_fixture("issue20.csv") + with open(test_file, "r+b") as f: + memory_mapped_file = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + data = get_data(memory_mapped_file, file_type="csv") + expected = [[u"to", u"infinity", u"and", u"beyond"]] + eq_(data["csv"], expected) def test_issue_30_utf8_BOM_header(): @@ -82,52 +76,31 @@ def test_issue_30_utf8_BOM_header(): def test_issue_33_34_utf32_encoded_file(): - if PY26: - pass - else: - check_mmap_encoding("utf-32") + check_mmap_encoding("utf-32") def test_issue_33_34_utf32be_encoded_file(): - if PY26: - pass - else: - check_mmap_encoding("utf-32-be") + check_mmap_encoding("utf-32-be") def test_issue_33_34_utf32le_encoded_file(): - if PY26: - pass - else: - check_mmap_encoding("utf-32-le") + check_mmap_encoding("utf-32-le") def test_issue_33_34_utf16_encoded_file(): - if PY26: - pass - else: - check_mmap_encoding("utf-16") + check_mmap_encoding("utf-16") def test_issue_33_34_utf16be_encoded_file(): - if PY26: - pass - else: - check_mmap_encoding("utf-16-be") + check_mmap_encoding("utf-16-be") def test_issue_33_34_utf16le_encoded_file(): - if PY26: - pass - else: - check_mmap_encoding("utf-16-le") + check_mmap_encoding("utf-16-le") def test_issue_33_34_utf8_encoded_file(): - if PY26: - pass - else: - check_mmap_encoding("utf-8") + check_mmap_encoding("utf-8") def check_mmap_encoding(encoding): diff --git a/tests/test_service.py b/tests/test_service.py index 5971708..8d0af9a 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,5 +1,4 @@ from pyexcel_io.service import ( - ODS_WRITE_FORMAT_COVERSION, date_value, time_value, ods_float_value, @@ -7,10 +6,8 @@ from pyexcel_io.service import ( detect_int_value, detect_float_value, ) -from pyexcel_io._compact import PY2 from pyexcel_io.exceptions import IntegerAccuracyLossError -from nose import SkipTest from nose.tools import eq_, raises @@ -101,34 +98,11 @@ def test_detect_float_value_on_custom_nan_text2(): eq_(str(result), "nan") -def test_ods_write_format_conversion(): - if PY2: - expected = ODS_WRITE_FORMAT_COVERSION[long] # noqa: F821 - eq_("long", expected) - else: - raise SkipTest() - - @raises(IntegerAccuracyLossError) def test_big_int_value(): ods_float_value(1000000000000000) -def test_max_value_on_python_2(): - if PY2: - ods_float_value(long(999999999999999)) - else: - raise SkipTest("No long in python 3") - - -@raises(IntegerAccuracyLossError) -def test_really_long_value_on_python2(): - if PY2: - ods_float_value(long(999999999999999 + 1)) - else: - raise SkipTest("No long in python 3") - - @raises(IntegerAccuracyLossError) def test_throw_exception(): throw_exception(1000000000000000) From a50b214e92461302d11ffbb9879d0fbc5117bdfa Mon Sep 17 00:00:00 2001 From: jaska Date: Sat, 26 Sep 2020 14:32:48 +0100 Subject: [PATCH 082/143] WIP - New style database plugin (#78) * :tada: new style django model reader * :tada: new style sqlalchemy table reader * :sparkles: new style django model importer * :sparkles: new style sql table importer * :fire: remove useless code * :lipstick: update programming style * :microscope: more test coverage --- pyexcel_io/database/__init__.py | 30 ++++++++++--- pyexcel_io/database/common.py | 18 -------- pyexcel_io/database/exporters/django.py | 25 +++++------ pyexcel_io/database/exporters/sqlalchemy.py | 23 +++++----- pyexcel_io/database/importers/django.py | 22 +++++++--- pyexcel_io/database/importers/sqlalchemy.py | 21 ++++++--- pyexcel_io/readers/csv_in_file.py | 4 +- pyexcel_io/readers/csv_in_memory.py | 4 +- pyexcel_io/writer.py | 4 +- tests/test_django_book.py | 47 +++++++-------------- tests/test_io.py | 6 +-- tests/test_sql_book.py | 25 ++++++----- 12 files changed, 115 insertions(+), 114 deletions(-) diff --git a/pyexcel_io/database/__init__.py b/pyexcel_io/database/__init__.py index 716c2e5..2e355f1 100644 --- a/pyexcel_io/database/__init__.py +++ b/pyexcel_io/database/__init__.py @@ -7,19 +7,39 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.plugins import IOPluginInfoChain +from pyexcel_io.plugins import NewIOPluginInfoChain from pyexcel_io.constants import DB_SQL, DB_DJANGO -IOPluginInfoChain(__name__).add_a_reader( +NewIOPluginInfoChain(__name__).add_a_reader( relative_plugin_class_path="exporters.django.DjangoBookReader", + location="file", + file_types=[DB_DJANGO], +).add_a_reader( + relative_plugin_class_path="exporters.django.DjangoBookReader", + location="memory", + file_types=[DB_DJANGO], +).add_a_reader( + relative_plugin_class_path="exporters.django.DjangoBookReader", + location="content", + file_types=[DB_DJANGO], +).add_a_writer( + relative_plugin_class_path="importers.django.DjangoBookWriter", + locations=["file", "content", "memory"], file_types=[DB_DJANGO], ).add_a_reader( relative_plugin_class_path="exporters.sqlalchemy.SQLBookReader", + location="file", + file_types=[DB_SQL], +).add_a_reader( + relative_plugin_class_path="exporters.sqlalchemy.SQLBookReader", + location="memory", + file_types=[DB_SQL], +).add_a_reader( + relative_plugin_class_path="exporters.sqlalchemy.SQLBookReader", + location="content", file_types=[DB_SQL], -).add_a_writer( - relative_plugin_class_path="importers.django.DjangoBookWriter", - file_types=[DB_DJANGO], ).add_a_writer( relative_plugin_class_path="importers.sqlalchemy.SQLBookWriter", + locations=["file", "content", "memory"], file_types=[DB_SQL], ) diff --git a/pyexcel_io/database/common.py b/pyexcel_io/database/common.py index b033fab..1f60ebd 100644 --- a/pyexcel_io/database/common.py +++ b/pyexcel_io/database/common.py @@ -7,24 +7,6 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.book import BookReader - - -class DbExporter(BookReader): - """ Transcode the book reader interface to db interface """ - - def open(self, file_name, **keywords): - self.export_tables(self, file_name, **keywords) - - def open_stream(self, file_stream, **keywords): - self.export_tables(self, file_stream, **keywords) - - def open_content(self, file_content, **keywords): - self.export_tables(file_content, **keywords) - - def export_tables(self, exporter, **keywords): - """ read database tables """ - raise NotImplementedError("Please implement this method") class DjangoModelExportAdapter(object): diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index 862c1e1..a3ff4d7 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -7,7 +7,6 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.database.common import DbExporter from pyexcel_io.database.querysets import QuerysetsReader @@ -27,22 +26,20 @@ class DjangoModelReader(QuerysetsReader): ) -class DjangoBookReader(DbExporter): +class DjangoBookReader(object): """ read django models """ - def __init__(self): - DbExporter.__init__(self) - self.exporter = None + def __init__(self, exporter, **keywords): + self.exporter = exporter + self.keywords = keywords + self.content_array = self.exporter.adapters - def export_tables(self, file_content, **keywords): - self.exporter = file_content - self._load_from_django_models() - - def read_sheet(self, native_sheet): + def read_sheet(self, native_sheet_index): + native_sheet = self.content_array[native_sheet_index] reader = DjangoModelReader( - native_sheet.model, native_sheet.export_columns + native_sheet.model, export_columns=native_sheet.export_columns ) - return reader.to_array() + return reader - def _load_from_django_models(self): - self._native_book = self.exporter.adapters + def close(self): + pass diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index 2ab2c8a..8f24c26 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -7,7 +7,6 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.database.common import DbExporter from pyexcel_io.database.querysets import QuerysetsReader @@ -31,24 +30,22 @@ class SQLTableReader(QuerysetsReader): QuerysetsReader.__init__(self, everything, column_names, **keywords) -class SQLBookReader(DbExporter): +class SQLBookReader(object): """ read a table via sqlalchemy """ - def __init__(self): - DbExporter.__init__(self) - self.__exporter = None + def __init__(self, exporter, **keywords): + self.__exporter = exporter + self.content_array = self.__exporter.adapters + self.keywords = keywords - def export_tables(self, file_content, **keywords): - self.__exporter = file_content - self._load_from_tables() - - def read_sheet(self, native_sheet): + def read_sheet(self, native_sheet_index): + native_sheet = self.content_array[native_sheet_index] reader = SQLTableReader( self.__exporter.session, native_sheet.table, native_sheet.export_columns, ) - return reader.to_array() + return reader - def _load_from_tables(self): - self._native_book = self.__exporter.adapters + def close(self): + pass diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index d6b266d..1be6445 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -58,15 +58,11 @@ class DjangoModelWriter(SheetWriter): an_object.save() -class DjangoBookWriter(BookWriter): +class DjangoBookWriter(object): """ write data into django models """ - def __init__(self): - BookWriter.__init__(self) - self.__importer = None - - def open_content(self, file_content, **keywords): - self.__importer = file_content + def __init__(self, exporter, **keywords): + self.__importer = exporter self._keywords = keywords def create_sheet(self, sheet_name): @@ -86,3 +82,15 @@ class DjangoBookWriter(BookWriter): ) return sheet_writer + + def write(self, incoming_dict): + for sheet_name in incoming_dict: + sheet_writer = self.create_sheet(sheet_name) + if sheet_writer: + sheet_writer.write_array(incoming_dict[sheet_name]) + sheet_writer.close() + else: + raise Exception("Cannot create a sheet writer!") + + def close(self): + pass diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 1e4c3c2..a2dd877 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -72,15 +72,10 @@ class SQLTableWriter(SheetWriter): self._native_book.session.commit() -class SQLBookWriter(BookWriter): +class SQLBookWriter(object): """ write data into database tables via sqlalchemy """ - def __init__(self): - BookWriter.__init__(self) - self.__importer = None - self.__auto_commit = True - - def open_content(self, file_content, auto_commit=True, **keywords): + def __init__(self, file_content, auto_commit=True, **keywords): self.__importer = file_content self.__auto_commit = auto_commit @@ -98,3 +93,15 @@ class SQLBookWriter(BookWriter): ) return sheet_writer + + def write(self, incoming_dict): + for sheet_name in incoming_dict: + sheet_writer = self.create_sheet(sheet_name) + if sheet_writer: + sheet_writer.write_array(incoming_dict[sheet_name]) + sheet_writer.close() + else: + raise Exception("Cannot create a sheet writer!") + + def close(self): + pass diff --git a/pyexcel_io/readers/csv_in_file.py b/pyexcel_io/readers/csv_in_file.py index fa68618..c9d6738 100644 --- a/pyexcel_io/readers/csv_in_file.py +++ b/pyexcel_io/readers/csv_in_file.py @@ -51,7 +51,9 @@ class FileReader(object): self.content_array = ret def read_sheet(self, index): - return CSVFileReader(self.content_array[index], **self.keywords) + reader = CSVFileReader(self.content_array[index], **self.keywords) + self.handles.append(reader) + return reader def close(self): for reader in self.handles: diff --git a/pyexcel_io/readers/csv_in_memory.py b/pyexcel_io/readers/csv_in_memory.py index aa54693..e64a5c9 100644 --- a/pyexcel_io/readers/csv_in_memory.py +++ b/pyexcel_io/readers/csv_in_memory.py @@ -48,7 +48,9 @@ class MemoryReader(object): self.content_array = [NamedContent(self.file_type, file_stream)] def read_sheet(self, index): - return CSVinMemoryReader(self.content_array[index], **self.keywords) + reader = CSVinMemoryReader(self.content_array[index], **self.keywords) + self.handles.append(reader) + return reader def close(self): for reader in self.handles: diff --git a/pyexcel_io/writer.py b/pyexcel_io/writer.py index 493f75b..806fe1b 100644 --- a/pyexcel_io/writer.py +++ b/pyexcel_io/writer.py @@ -17,8 +17,8 @@ class Writer(object): self.writer = writer_class(file_name, **keywords) def open_content(self, file_stream, **keywords): - if not isstream(file_stream): - raise IOError(MESSAGE_ERROR_03) + # if not isstream(file_stream): + # raise IOError(MESSAGE_ERROR_03) writer_class = NEW_WRITERS.get_a_plugin( self.file_type, library=self.library, location="content" ) diff --git a/tests/test_django_book.py b/tests/test_django_book.py index 7ad4a6a..3fc320a 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -280,8 +280,7 @@ class TestMultipleModels: adapter1.get_name(): self.content["Sheet1"][1:], adapter2.get_name(): self.content["Sheet2"][1:], } - writer = DjangoBookWriter() - writer.open_content(importer, batch_size=sample_size) + writer = DjangoBookWriter(importer, batch_size=sample_size) writer.write(to_store) writer.close() assert model1.objects.objs == self.result1 @@ -303,8 +302,9 @@ class TestMultipleModels: adapter1.get_name(): self.content["Sheet1"][1:], adapter2.get_name(): self.content["Sheet2"][1:], } - writer = DjangoBookWriter() - writer.open_content(importer, batch_size=sample_size, bulk_save=False) + writer = DjangoBookWriter( + importer, batch_size=sample_size, bulk_save=False + ) writer.write(to_store) writer.close() assert model1.objects.objs == [] @@ -335,12 +335,17 @@ class TestMultipleModels: adapter2 = DjangoModelExportAdapter(model2) exporter.append(adapter1) exporter.append(adapter2) - reader = DjangoBookReader() - reader.open_content(exporter) - data = reader.read_all() - for key in data.keys(): - data[key] = list(data[key]) - assert data == self.content + reader = DjangoBookReader(exporter) + result = OrderedDict() + for index, sheet in enumerate(reader.content_array): + result.update( + { + reader.content_array[index].name: list( + reader.read_sheet(index).to_array() + ) + } + ) + assert result == self.content @raises(Exception) def test_special_case_where_only_one_model_used(self): @@ -354,28 +359,6 @@ class TestMultipleModels: "Sheet2": self.content["Sheet2"][1:], } save_data(importer, to_store, file_type=DB_DJANGO) - assert model1.objects.objs == self.result1 - model1._meta.model_name = "Sheet1" - model1._meta.update(["X", "Y", "Z"]) - exporter = DjangoModelExporter() - adapter = DjangoModelExportAdapter(model1) - exporter.append(adapter) - reader = DjangoBookReader() - reader.open_content(exporter) - data = reader.read_all() - assert list(data["Sheet1"]) == self.content["Sheet1"] - - -@raises(TypeError) -def test_not_implemented_method(): - reader = DjangoBookReader() - reader.open("afile") - - -@raises(TypeError) -def test_not_implemented_method_2(): - reader = DjangoBookReader() - reader.open_stream("afile") class TestFilter: diff --git a/tests/test_io.py b/tests/test_io.py index 0646244..067754a 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -68,9 +68,9 @@ def test_wrong_parameter_to_get_writer(): get_writer(1) -@raises(Exception) -def test_wrong_parameter_to_get_writer2(): - get_writer(1, file_type="csv") +# @raises(Exception) +# def test_wrong_parameter_to_get_writer2(): +# get_writer(1, file_type="csv") def test_load_ods_data(): diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index cba8177..8533e57 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -441,8 +441,7 @@ class TestMultipleRead: post_adapter.column_names = data["Post"][0] post_adapter.row_initializer = post_init_func importer.append(post_adapter) - writer = SQLBookWriter() - writer.open_content(importer) + writer = SQLBookWriter(importer) to_store = OrderedDict() to_store.update({category_adapter.get_name(): data["Category"][1:]}) to_store.update({post_adapter.get_name(): data["Post"][1:]}) @@ -455,12 +454,17 @@ class TestMultipleRead: exporter.append(category_adapter) post_adapter = SQLTableExportAdapter(Post) exporter.append(post_adapter) - book = SQLBookReader() - book.open_content(exporter) - data = book.read_all() - for key in data.keys(): - data[key] = list(data[key]) - assert json.dumps(data) == ( + reader = SQLBookReader(exporter) + result = OrderedDict() + for index, sheet in enumerate(reader.content_array): + result.update( + { + reader.content_array[index].name: list( + reader.read_sheet(index).to_array() + ) + } + ) + assert json.dumps(result) == ( '{"category": [["id", "name"], [1, "News"], [2, "Sports"]], ' + '"post": [["body", "category_id", "id", "pub_date", "title"], ' + '["formal", 1, 1, "2015-01-20T23:28:29", "Title A"], ' @@ -550,13 +554,12 @@ def test_sql_table_import_adapter(): @raises(Exception) -def test_unknown_sheet(self): +def test_unknown_sheet(): importer = SQLTableImporter(None) category_adapter = SQLTableImportAdapter(Category) category_adapter.column_names = [""] importer.append(category_adapter) - writer = SQLBookWriter() - writer.open_content(importer) + writer = SQLBookWriter(importer) to_store = OrderedDict() to_store.update({"you do not see me": [[]]}) writer.write(to_store) From 9fa3f4051e64a504db1b2726555e2fe42466054f Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 26 Sep 2020 13:33:28 +0000 Subject: [PATCH 083/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 984a923..da577f8 100644 --- a/README.rst +++ b/README.rst @@ -72,8 +72,10 @@ sqlalchemy supported databases. Its supported file formats are extended to cover ======================== ======================= ================= ================== Package name Supported file formats Dependencies Python versions ======================== ======================= ================= ================== - `pyexcel-io`_ csv, csvz [#f1]_, tsv, 2.6, 2.7, 3.3, - tsvz [#f2]_ 3.4, 3.5, 3.6 + `pyexcel-io`_ >=v0.6.0 csv, csvz [#f1]_, tsv, 3.6+ + tsvz [#f2]_ + `pyexcel-io`_ <=0.5.20 same as above 2.6, 2.7, 3.3, + 3.4, 3.5, 3.6 pypy `pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, same as above xlsm(read only) `xlwt`_ From 093c937773f4b1f7a36c3c47645218b3ad95dce7 Mon Sep 17 00:00:00 2001 From: jaska Date: Sat, 26 Sep 2020 15:00:19 +0100 Subject: [PATCH 084/143] plugin and writer interfaces update (#79) * :tractor: update new io plugin adder * :hammer: interface update * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst Co-authored-by: chfw --- lint.sh | 3 +-- pyexcel_io/database/__init__.py | 20 ++-------------- pyexcel_io/database/importers/django.py | 1 - pyexcel_io/database/importers/sqlalchemy.py | 1 - pyexcel_io/plugins.py | 8 +++++-- pyexcel_io/readers/__init__.py | 26 +++++++-------------- pyexcel_io/writer.py | 11 +-------- pyexcel_io/writers/csv_in_file.py | 9 +++++++ pyexcel_io/writers/csv_in_memory.py | 9 +++++++ pyexcel_io/writers/csvz_writer.py | 9 +++++++ tests/test_book.py | 1 - 11 files changed, 45 insertions(+), 53 deletions(-) diff --git a/lint.sh b/lint.sh index 6907d07..742144b 100644 --- a/lint.sh +++ b/lint.sh @@ -1,3 +1,2 @@ pip install flake8 -flake8 . --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long -python setup.py checkdocs +flake8 . --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long && python setup.py checkdocs diff --git a/pyexcel_io/database/__init__.py b/pyexcel_io/database/__init__.py index 2e355f1..e5763f7 100644 --- a/pyexcel_io/database/__init__.py +++ b/pyexcel_io/database/__init__.py @@ -12,15 +12,7 @@ from pyexcel_io.constants import DB_SQL, DB_DJANGO NewIOPluginInfoChain(__name__).add_a_reader( relative_plugin_class_path="exporters.django.DjangoBookReader", - location="file", - file_types=[DB_DJANGO], -).add_a_reader( - relative_plugin_class_path="exporters.django.DjangoBookReader", - location="memory", - file_types=[DB_DJANGO], -).add_a_reader( - relative_plugin_class_path="exporters.django.DjangoBookReader", - location="content", + locations=["file", "memory", "content"], file_types=[DB_DJANGO], ).add_a_writer( relative_plugin_class_path="importers.django.DjangoBookWriter", @@ -28,15 +20,7 @@ NewIOPluginInfoChain(__name__).add_a_reader( file_types=[DB_DJANGO], ).add_a_reader( relative_plugin_class_path="exporters.sqlalchemy.SQLBookReader", - location="file", - file_types=[DB_SQL], -).add_a_reader( - relative_plugin_class_path="exporters.sqlalchemy.SQLBookReader", - location="memory", - file_types=[DB_SQL], -).add_a_reader( - relative_plugin_class_path="exporters.sqlalchemy.SQLBookReader", - location="content", + locations=["file", "memory", "content"], file_types=[DB_SQL], ).add_a_writer( relative_plugin_class_path="importers.sqlalchemy.SQLBookWriter", diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 1be6445..4c055be 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -10,7 +10,6 @@ import logging import pyexcel_io.constants as constants -from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetWriter from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index a2dd877..627fa07 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -8,7 +8,6 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants -from pyexcel_io.book import BookWriter from pyexcel_io.sheet import SheetWriter from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 89b0e6e..9f3c8aa 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -71,7 +71,7 @@ class NewIOPluginInfoChain(PluginInfoChain): def add_a_reader( self, relative_plugin_class_path=None, - location="file", + locations=(), file_types=None, stream_type=None, ): @@ -79,7 +79,11 @@ class NewIOPluginInfoChain(PluginInfoChain): a_plugin_info = IOPluginInfo( NEW_READER_PLUGIN, self._get_abs_path(relative_plugin_class_path), - file_types=[f"{location}-{file_type}" for file_type in file_types], + file_types=[ + f"{location}-{file_type}" + for file_type in file_types + for location in locations + ], stream_type=stream_type, ) return self.add_a_plugin_instance(a_plugin_info) diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index 9cb3cb2..d3af47f 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -11,52 +11,42 @@ from pyexcel_io.plugins import NewIOPluginInfoChain NewIOPluginInfoChain(__name__).add_a_reader( relative_plugin_class_path="csv_in_file.FileReader", - location="file", + locations=["file"], file_types=["csv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="csv_content.ContentReader", - location="content", + locations=["content"], file_types=["csv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="csv_in_memory.MemoryReader", - location="memory", + locations=["memory"], file_types=["csv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="tsv.TSVMemoryReader", - location="memory", + locations=["memory"], file_types=["tsv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="tsv.TSVFileReader", - location="file", + locations=["file"], file_types=["tsv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="tsv.TSVContentReader", - location="content", + locations=["content"], file_types=["tsv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="csvz.FileReader", file_types=["csvz"], - location="file", - stream_type="binary", -).add_a_reader( - relative_plugin_class_path="csvz.FileReader", - file_types=["csvz"], - location="memory", + locations=["file", "memory"], stream_type="binary", ).add_a_reader( relative_plugin_class_path="tsvz.TSVZipFileReader", file_types=["tsvz"], - location="file", - stream_type="binary", -).add_a_reader( - relative_plugin_class_path="tsvz.TSVZipFileReader", - file_types=["tsvz"], - location="memory", + locations=["file", "memory"], stream_type="binary", ) diff --git a/pyexcel_io/writer.py b/pyexcel_io/writer.py index 806fe1b..16f2085 100644 --- a/pyexcel_io/writer.py +++ b/pyexcel_io/writer.py @@ -1,7 +1,4 @@ from pyexcel_io.plugins import NEW_WRITERS -from pyexcel_io._compact import isstream - -from .constants import MESSAGE_ERROR_03 class Writer(object): @@ -31,13 +28,7 @@ class Writer(object): self.writer = writer_class(file_stream, **keywords) def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.writer.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") + self.writer.write(incoming_dict) def close(self): self.writer.close() diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py index f093cdf..9df63fd 100644 --- a/pyexcel_io/writers/csv_in_file.py +++ b/pyexcel_io/writers/csv_in_file.py @@ -18,6 +18,15 @@ class CsvFileWriter: self.__index = self.__index + 1 return self.writer + def write(self, incoming_dict): + for sheet_name in incoming_dict: + sheet_writer = self.create_sheet(sheet_name) + if sheet_writer: + sheet_writer.write_array(incoming_dict[sheet_name]) + sheet_writer.close() + else: + raise Exception("Cannot create a sheet writer!") + def close(self): if self.writer: self.writer.close() diff --git a/pyexcel_io/writers/csv_in_memory.py b/pyexcel_io/writers/csv_in_memory.py index a3ad8b2..b4f84bd 100644 --- a/pyexcel_io/writers/csv_in_memory.py +++ b/pyexcel_io/writers/csv_in_memory.py @@ -18,5 +18,14 @@ class CsvMemoryWriter: self.__index = self.__index + 1 return writer + def write(self, incoming_dict): + for sheet_name in incoming_dict: + sheet_writer = self.create_sheet(sheet_name) + if sheet_writer: + sheet_writer.write_array(incoming_dict[sheet_name]) + sheet_writer.close() + else: + raise Exception("Cannot create a sheet writer!") + def close(self): pass diff --git a/pyexcel_io/writers/csvz_writer.py b/pyexcel_io/writers/csvz_writer.py index 19bffff..cf80621 100644 --- a/pyexcel_io/writers/csvz_writer.py +++ b/pyexcel_io/writers/csvz_writer.py @@ -27,6 +27,15 @@ class CsvZipWriter(object): ) return writer + def write(self, incoming_dict): + for sheet_name in incoming_dict: + sheet_writer = self.create_sheet(sheet_name) + if sheet_writer: + sheet_writer.write_array(incoming_dict[sheet_name]) + sheet_writer.close() + else: + raise Exception("Cannot create a sheet writer!") + def close(self): if self.zipfile: self.zipfile.close() diff --git a/tests/test_book.py b/tests/test_book.py index c14a380..bb6b20e 100644 --- a/tests/test_book.py +++ b/tests/test_book.py @@ -6,7 +6,6 @@ from pyexcel_io.book import ( ) from pyexcel_io._compact import BytesIO, StringIO -from nose import SkipTest from nose.tools import raises From f6293637e8f951d93b42c268b7518e33d5fcf24e Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 26 Sep 2020 22:36:22 +0100 Subject: [PATCH 085/143] :bug: unwanted hard code for it to work with existing code. this fix shall go to pyexcel, but not here. let do it properly in next version --- pyexcel_io/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 89b0e6e..5bbb787 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -227,14 +227,14 @@ class AllReaders: def get_all_formats(self): return OLD_READERS.get_all_formats().union( NEW_READERS.get_all_formats() - ) + ) - set([constants.DB_SQL, constants.DB_DJANGO]) class AllWriters: def get_all_formats(self): return OLD_WRITERS.get_all_formats().union( NEW_WRITERS.get_all_formats() - ) + ) - set([constants.DB_SQL, constants.DB_DJANGO]) OLD_READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS) From b2c774cd94954d009b6af5156b4a82749384abf2 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 26 Sep 2020 21:45:17 +0000 Subject: [PATCH 086/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- Makefile | 3 --- lint.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d7e640e..3e0ee51 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,6 @@ test: lint install_test: pip install -r tests/requirements.txt -git-diff-check: - git diff --exit-code - lint: bash lint.sh diff --git a/lint.sh b/lint.sh index 742144b..891aa63 100644 --- a/lint.sh +++ b/lint.sh @@ -1,2 +1,2 @@ pip install flake8 -flake8 . --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long && python setup.py checkdocs +flake8 --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long . && python setup.py checkdocs From 020b8f0a9fa41ec22a7997a6229c1695936d9551 Mon Sep 17 00:00:00 2001 From: jaska Date: Sun, 27 Sep 2020 23:30:53 +0100 Subject: [PATCH 087/143] One reader/writer handle many file types, i.e. csv reader for csv and tsv (#81) * :bug: pyexcel-io used to allow one reader/writer handles more than one file type * :books: update pyinstaller instructions * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :lipstick: update coding style Co-authored-by: chfw --- CHANGELOG.rst | 2 +- changelog.yml | 3 +- docs/source/pyinstaller.rst | 5 --- pyexcel_io/database/exporters/django.py | 2 +- pyexcel_io/database/exporters/sqlalchemy.py | 2 +- pyexcel_io/database/importers/django.py | 2 +- pyexcel_io/database/importers/sqlalchemy.py | 2 +- pyexcel_io/reader.py | 12 +++++-- pyexcel_io/readers/__init__.py | 28 +++------------ pyexcel_io/readers/csv_content.py | 9 ++--- pyexcel_io/readers/csv_in_file.py | 4 ++- pyexcel_io/readers/csv_in_memory.py | 10 ++++-- pyexcel_io/readers/csvz.py | 5 ++- pyexcel_io/readers/tsv.py | 38 --------------------- pyexcel_io/readers/tsvz.py | 22 ------------ pyexcel_io/writer.py | 6 ++-- pyexcel_io/writers/__init__.py | 21 ++---------- pyexcel_io/writers/csv_in_file.py | 5 ++- pyexcel_io/writers/csv_in_memory.py | 5 ++- pyexcel_io/writers/csvz_writer.py | 10 +++--- pyexcel_io/writers/tsv_in_file.py | 9 ----- pyexcel_io/writers/tsv_in_memory.py | 9 ----- pyexcel_io/writers/tsvz_writer.py | 8 ----- tests/test_django_book.py | 6 ++-- tests/test_sql_book.py | 6 ++-- 25 files changed, 63 insertions(+), 168 deletions(-) delete mode 100644 pyexcel_io/readers/tsv.py delete mode 100644 pyexcel_io/readers/tsvz.py delete mode 100644 pyexcel_io/writers/tsv_in_file.py delete mode 100644 pyexcel_io/writers/tsv_in_memory.py delete mode 100644 pyexcel_io/writers/tsvz_writer.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fbc5526..37abb53 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,7 +16,7 @@ Change log **updated** #. pyexcel-io plugin interface has been rewritten. PyInstaller user will be - impacted. + impacted. please read 'Packaging with Pyinstaller' in the documentation. 0.5.20 - 17.7.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index 62a544f..58ce90b 100644 --- a/changelog.yml +++ b/changelog.yml @@ -10,7 +10,8 @@ releases: - 'python 3.6 lower versions are no longer supported' - action: updated details: - - 'pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted.' + - pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted. + please read 'Packaging with Pyinstaller' in the documentation. version: 0.6.0 date: tbd - changes: diff --git a/docs/source/pyinstaller.rst b/docs/source/pyinstaller.rst index d92179a..2db1c36 100644 --- a/docs/source/pyinstaller.rst +++ b/docs/source/pyinstaller.rst @@ -13,14 +13,9 @@ In order to package every built-in plugins of pyexcel-io, you need to specify:: --hidden-import pyexcel_io.readers.csv_in_memory --hidden-import pyexcel_io.readers.csv_content --hidden-import pyexcel_io.readers.csvz - --hidden-import pyexcel_io.readers.tsv - --hidden-import pyexcel_io.readers.tsvz --hidden-import pyexcel_io.writers.csv_in_file --hidden-import pyexcel_io.writers.csv_in_memory - --hidden-import pyexcel_io.writers.tsv_in_file - --hidden-import pyexcel_io.writers.tsv_in_memory --hidden-import pyexcel_io.writers.csvz_writer - --hidden-import pyexcel_io.writers.tsvz_writer --hidden-import pyexcel_io.database.importers.django --hidden-import pyexcel_io.database.importers.sqlalchemy --hidden-import pyexcel_io.database.exporters.django diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index a3ff4d7..0717cfd 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -29,7 +29,7 @@ class DjangoModelReader(QuerysetsReader): class DjangoBookReader(object): """ read django models """ - def __init__(self, exporter, **keywords): + def __init__(self, exporter, _, **keywords): self.exporter = exporter self.keywords = keywords self.content_array = self.exporter.adapters diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index 8f24c26..077cb21 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -33,7 +33,7 @@ class SQLTableReader(QuerysetsReader): class SQLBookReader(object): """ read a table via sqlalchemy """ - def __init__(self, exporter, **keywords): + def __init__(self, exporter, _, **keywords): self.__exporter = exporter self.content_array = self.__exporter.adapters self.keywords = keywords diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 4c055be..0a1f5d2 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -60,7 +60,7 @@ class DjangoModelWriter(SheetWriter): class DjangoBookWriter(object): """ write data into django models """ - def __init__(self, exporter, **keywords): + def __init__(self, exporter, _, **keywords): self.__importer = exporter self._keywords = keywords diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 627fa07..c51034f 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -74,7 +74,7 @@ class SQLTableWriter(SheetWriter): class SQLBookWriter(object): """ write data into database tables via sqlalchemy """ - def __init__(self, file_content, auto_commit=True, **keywords): + def __init__(self, file_content, _, auto_commit=True, **keywords): self.__importer = file_content self.__auto_commit = auto_commit diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py index 0d9a9aa..49d08c3 100644 --- a/pyexcel_io/reader.py +++ b/pyexcel_io/reader.py @@ -37,7 +37,9 @@ class Reader(object): self.file_type, location="file", library=self.library ) self.keywords, native_sheet_keywords = clean_keywords(keywords) - self.reader = reader_class(file_name, **native_sheet_keywords) + self.reader = reader_class( + file_name, self.file_type, **native_sheet_keywords + ) return self.reader def open_content(self, file_content, **keywords): @@ -46,7 +48,9 @@ class Reader(object): reader_class = NEW_READERS.get_a_plugin( self.file_type, location="content", library=self.library ) - self.reader = reader_class(file_content, **native_sheet_keywords) + self.reader = reader_class( + file_content, self.file_type, **native_sheet_keywords + ) return self.reader except ( exceptions.NoSupportingPluginFound, @@ -62,7 +66,9 @@ class Reader(object): reader_class = NEW_READERS.get_a_plugin( self.file_type, location="memory", library=self.library ) - self.reader = reader_class(file_stream, **native_sheet_keywords) + self.reader = reader_class( + file_stream, self.file_type, **native_sheet_keywords + ) return self.reader def read_sheet_by_name(self, sheet_name): diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index d3af47f..c0be33d 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -12,41 +12,21 @@ from pyexcel_io.plugins import NewIOPluginInfoChain NewIOPluginInfoChain(__name__).add_a_reader( relative_plugin_class_path="csv_in_file.FileReader", locations=["file"], - file_types=["csv"], + file_types=["csv", "tsv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="csv_content.ContentReader", locations=["content"], - file_types=["csv"], + file_types=["csv", "tsv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="csv_in_memory.MemoryReader", locations=["memory"], - file_types=["csv"], - stream_type="text", -).add_a_reader( - relative_plugin_class_path="tsv.TSVMemoryReader", - locations=["memory"], - file_types=["tsv"], - stream_type="text", -).add_a_reader( - relative_plugin_class_path="tsv.TSVFileReader", - locations=["file"], - file_types=["tsv"], - stream_type="text", -).add_a_reader( - relative_plugin_class_path="tsv.TSVContentReader", - locations=["content"], - file_types=["tsv"], + file_types=["csv", "tsv"], stream_type="text", ).add_a_reader( relative_plugin_class_path="csvz.FileReader", - file_types=["csvz"], - locations=["file", "memory"], - stream_type="binary", -).add_a_reader( - relative_plugin_class_path="tsvz.TSVZipFileReader", - file_types=["tsvz"], + file_types=["csvz", "tsvz"], locations=["file", "memory"], stream_type="binary", ) diff --git a/pyexcel_io/readers/csv_content.py b/pyexcel_io/readers/csv_content.py index 96636cf..21a96c1 100644 --- a/pyexcel_io/readers/csv_content.py +++ b/pyexcel_io/readers/csv_content.py @@ -1,19 +1,16 @@ import mmap -import pyexcel_io.constants as constants from pyexcel_io.book import _convert_content_to_stream from pyexcel_io.readers.csv_sheet import CSVMemoryMapIterator from pyexcel_io.readers.csv_in_memory import MemoryReader class ContentReader(MemoryReader): - file_type = constants.FILE_FORMAT_CSV - - def __init__(self, file_content, **keywords): + def __init__(self, file_content, file_type, **keywords): file_stream = ContentReader.convert_content_to_stream( - file_content, self.file_type, **keywords + file_content, file_type, **keywords ) - super().__init__(file_stream, **keywords) + super().__init__(file_stream, file_type, **keywords) @staticmethod def convert_content_to_stream(file_content, file_type, **keywords): diff --git a/pyexcel_io/readers/csv_in_file.py b/pyexcel_io/readers/csv_in_file.py index c9d6738..535f8be 100644 --- a/pyexcel_io/readers/csv_in_file.py +++ b/pyexcel_io/readers/csv_in_file.py @@ -10,13 +10,15 @@ DEFAULT_NEWLINE = "\r\n" class FileReader(object): - def __init__(self, file_name, **keywords): + def __init__(self, file_name, file_type, **keywords): """Load content from a file :params str filename: an accessible file path :returns: a book """ self.handles = [] self.keywords = keywords + if file_type == constants.FILE_FORMAT_TSV: + self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT self.__line_terminator = keywords.get( constants.KEYWORD_LINE_TERMINATOR, DEFAULT_NEWLINE ) diff --git a/pyexcel_io/readers/csv_in_memory.py b/pyexcel_io/readers/csv_in_memory.py index e64a5c9..3e39521 100644 --- a/pyexcel_io/readers/csv_in_memory.py +++ b/pyexcel_io/readers/csv_in_memory.py @@ -9,15 +9,19 @@ DEFAULT_SHEET_SEPARATOR_FORMATTER = f"---{constants.DEFAULT_NAME}---%s" class MemoryReader(object): - file_type = constants.FILE_FORMAT_CSV - - def __init__(self, file_stream, multiple_sheets=False, **keywords): + def __init__( + self, file_stream, file_type, multiple_sheets=False, **keywords + ): """Load content from memory :params stream file_content: the actual file content in memory :returns: a book """ self.handles = [] self.keywords = keywords + if file_type == constants.FILE_FORMAT_TSV: + self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT + self.file_type = file_type + self.__load_from_memory_flag = True self.__line_terminator = keywords.get( constants.KEYWORD_LINE_TERMINATOR, constants.DEFAULT_CSV_NEWLINE diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index a783071..f435afd 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -10,13 +10,14 @@ import zipfile import chardet +from pyexcel_io import constants from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import StringIO from pyexcel_io.readers.csv_sheet import CSVinMemoryReader class FileReader(object): - def __init__(self, file_alike_object, **keywords): + def __init__(self, file_alike_object, file_type, **keywords): self.content_array = [] try: self.zipfile = zipfile.ZipFile(file_alike_object, "r") @@ -26,6 +27,8 @@ class FileReader(object): ] self.content_array = sheets self.keywords = keywords + if file_type == constants.FILE_FORMAT_TSVZ: + self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT except zipfile.BadZipfile: print("StringIO instance was passed by any chance?") diff --git a/pyexcel_io/readers/tsv.py b/pyexcel_io/readers/tsv.py deleted file mode 100644 index 79edea9..0000000 --- a/pyexcel_io/readers/tsv.py +++ /dev/null @@ -1,38 +0,0 @@ -""" - pyexcel_io.readers.tsv - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - The lower level tsv file format handler. - - :copyright: (c) 2014-2020 by Onni Software Ltd. - :license: New BSD License, see LICENSE for more details -""" -import pyexcel_io.constants as constants -from pyexcel_io.readers.csv_content import ContentReader -from pyexcel_io.readers.csv_in_file import FileReader -from pyexcel_io.readers.csv_in_memory import MemoryReader - - -class TSVFileReader(FileReader): - def __init__(self, file_name, **keywords): - super().__init__( - file_name, dialect=constants.KEYWORD_TSV_DIALECT, **keywords - ) - - -class TSVMemoryReader(MemoryReader): - file_type = constants.FILE_FORMAT_TSV - - def __init__(self, file_stream, **keywords): - super().__init__( - file_stream, dialect=constants.KEYWORD_TSV_DIALECT, **keywords - ) - - -class TSVContentReader(ContentReader): - file_type = constants.FILE_FORMAT_TSV - - def __init__(self, file_content, **keywords): - super().__init__( - file_content, dialect=constants.KEYWORD_TSV_DIALECT, **keywords - ) diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py deleted file mode 100644 index bd6d904..0000000 --- a/pyexcel_io/readers/tsvz.py +++ /dev/null @@ -1,22 +0,0 @@ -""" - pyexcel_io.fileformat.tsvz - ~~~~~~~~~~~~~~~~~~~~~~~~~~ - - The lower level tsvz file format handler. - - :copyright: (c) 2014-2020 by Onni Software Ltd. - :license: New BSD License, see LICENSE for more details -""" -from pyexcel_io.constants import KEYWORD_TSV_DIALECT - -from .csvz import FileReader - - -class TSVZipFileReader(FileReader): - """read zipped tab separated value file - - it supports single tsv file and mulitple tsv files - """ - - def __init__(self, file_name, **keywords): - super().__init__(file_name, dialect=KEYWORD_TSV_DIALECT, **keywords) diff --git a/pyexcel_io/writer.py b/pyexcel_io/writer.py index 16f2085..700d86e 100644 --- a/pyexcel_io/writer.py +++ b/pyexcel_io/writer.py @@ -11,7 +11,7 @@ class Writer(object): writer_class = NEW_WRITERS.get_a_plugin( self.file_type, library=self.library, location="file" ) - self.writer = writer_class(file_name, **keywords) + self.writer = writer_class(file_name, self.file_type, **keywords) def open_content(self, file_stream, **keywords): # if not isstream(file_stream): @@ -19,13 +19,13 @@ class Writer(object): writer_class = NEW_WRITERS.get_a_plugin( self.file_type, library=self.library, location="content" ) - self.writer = writer_class(file_stream, **keywords) + self.writer = writer_class(file_stream, self.file_type, **keywords) def open_stream(self, file_stream, **keywords): writer_class = NEW_WRITERS.get_a_plugin( self.file_type, library=self.library, location="memory" ) - self.writer = writer_class(file_stream, **keywords) + self.writer = writer_class(file_stream, self.file_type, **keywords) def write(self, incoming_dict): self.writer.write(incoming_dict) diff --git a/pyexcel_io/writers/__init__.py b/pyexcel_io/writers/__init__.py index e76dc29..89c9431 100644 --- a/pyexcel_io/writers/__init__.py +++ b/pyexcel_io/writers/__init__.py @@ -12,31 +12,16 @@ from pyexcel_io.plugins import NewIOPluginInfoChain NewIOPluginInfoChain(__name__).add_a_writer( relative_plugin_class_path="csv_in_file.CsvFileWriter", locations=["file", "content"], - file_types=["csv"], + file_types=["csv", "tsv"], stream_type="text", ).add_a_writer( relative_plugin_class_path="csv_in_memory.CsvMemoryWriter", locations=["memory"], - file_types=["csv"], - stream_type="text", -).add_a_writer( - relative_plugin_class_path="tsv_in_file.TsvFileWriter", - locations=["file", "content"], - file_types=["tsv"], - stream_type="text", -).add_a_writer( - relative_plugin_class_path="tsv_in_memory.TsvMemoryWriter", - locations=["memory"], - file_types=["tsv"], + file_types=["csv", "tsv"], stream_type="text", ).add_a_writer( relative_plugin_class_path="csvz_writer.CsvZipWriter", locations=["memory", "file", "content"], - file_types=["csvz"], - stream_type="binary", -).add_a_writer( - relative_plugin_class_path="tsvz_writer.TsvZipWriter", - locations=["memory", "file", "content"], - file_types=["tsvz"], + file_types=["csvz", "tsvz"], stream_type="binary", ) diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py index 9df63fd..111b74c 100644 --- a/pyexcel_io/writers/csv_in_file.py +++ b/pyexcel_io/writers/csv_in_file.py @@ -1,10 +1,13 @@ +from pyexcel_io import constants from pyexcel_io.writers.csv_sheet import CSVFileWriter class CsvFileWriter: - def __init__(self, file_alike_object, **keywords): + def __init__(self, file_alike_object, file_type, **keywords): self._file_alike_object = file_alike_object self._keywords = keywords + if file_type == constants.FILE_FORMAT_TSV: + self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT self.__index = 0 self.writer = None diff --git a/pyexcel_io/writers/csv_in_memory.py b/pyexcel_io/writers/csv_in_memory.py index b4f84bd..9e04b9d 100644 --- a/pyexcel_io/writers/csv_in_memory.py +++ b/pyexcel_io/writers/csv_in_memory.py @@ -1,10 +1,13 @@ +from pyexcel_io import constants from pyexcel_io.writers.csv_sheet import CSVMemoryWriter class CsvMemoryWriter: - def __init__(self, file_alike_object, **keywords): + def __init__(self, file_alike_object, file_type, **keywords): self._file_alike_object = file_alike_object self._keywords = keywords + if file_type == constants.FILE_FORMAT_TSV: + self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT self.__index = 0 def create_sheet(self, name): diff --git a/pyexcel_io/writers/csvz_writer.py b/pyexcel_io/writers/csvz_writer.py index cf80621..ace68c9 100644 --- a/pyexcel_io/writers/csvz_writer.py +++ b/pyexcel_io/writers/csvz_writer.py @@ -1,6 +1,6 @@ import zipfile -from pyexcel_io.constants import FILE_FORMAT_CSVZ, DEFAULT_SHEET_NAME +from pyexcel_io import constants from pyexcel_io.writers.csvz_sheet import CSVZipSheetWriter @@ -13,15 +13,17 @@ class CsvZipWriter(object): any other unzip software. """ - def __init__(self, file_name, **keywords): - self._file_type = FILE_FORMAT_CSVZ + def __init__(self, file_name, file_type, **keywords): + self._file_type = file_type self.zipfile = zipfile.ZipFile(file_name, "w", zipfile.ZIP_DEFLATED) self._keywords = keywords + if file_type == constants.FILE_FORMAT_TSVZ: + self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT def create_sheet(self, name): given_name = name if given_name is None: - given_name = DEFAULT_SHEET_NAME + given_name = constants.DEFAULT_SHEET_NAME writer = CSVZipSheetWriter( self.zipfile, given_name, self._file_type[:3], **self._keywords ) diff --git a/pyexcel_io/writers/tsv_in_file.py b/pyexcel_io/writers/tsv_in_file.py deleted file mode 100644 index 853285d..0000000 --- a/pyexcel_io/writers/tsv_in_file.py +++ /dev/null @@ -1,9 +0,0 @@ -from pyexcel_io.constants import KEYWORD_TSV_DIALECT -from pyexcel_io.writers.csv_in_file import CsvFileWriter - - -class TsvFileWriter(CsvFileWriter): - def __init__(self, file_alike_object, **keywords): - super().__init__( - file_alike_object, dialect=KEYWORD_TSV_DIALECT, **keywords - ) diff --git a/pyexcel_io/writers/tsv_in_memory.py b/pyexcel_io/writers/tsv_in_memory.py deleted file mode 100644 index 0e1d305..0000000 --- a/pyexcel_io/writers/tsv_in_memory.py +++ /dev/null @@ -1,9 +0,0 @@ -from pyexcel_io.constants import KEYWORD_TSV_DIALECT -from pyexcel_io.writers.csv_in_memory import CsvMemoryWriter - - -class TsvMemoryWriter(CsvMemoryWriter): - def __init__(self, file_alike_object, **keywords): - super().__init__( - file_alike_object, dialect=KEYWORD_TSV_DIALECT, **keywords - ) diff --git a/pyexcel_io/writers/tsvz_writer.py b/pyexcel_io/writers/tsvz_writer.py deleted file mode 100644 index 7eb438d..0000000 --- a/pyexcel_io/writers/tsvz_writer.py +++ /dev/null @@ -1,8 +0,0 @@ -from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT -from pyexcel_io.writers.csvz_writer import CsvZipWriter - - -class TsvZipWriter(CsvZipWriter): - def __init__(self, file_name, **keywords): - super().__init__(file_name, dialect=KEYWORD_TSV_DIALECT, **keywords) - self._file_type = FILE_FORMAT_TSVZ diff --git a/tests/test_django_book.py b/tests/test_django_book.py index 3fc320a..b92356c 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -280,7 +280,7 @@ class TestMultipleModels: adapter1.get_name(): self.content["Sheet1"][1:], adapter2.get_name(): self.content["Sheet2"][1:], } - writer = DjangoBookWriter(importer, batch_size=sample_size) + writer = DjangoBookWriter(importer, "django", batch_size=sample_size) writer.write(to_store) writer.close() assert model1.objects.objs == self.result1 @@ -303,7 +303,7 @@ class TestMultipleModels: adapter2.get_name(): self.content["Sheet2"][1:], } writer = DjangoBookWriter( - importer, batch_size=sample_size, bulk_save=False + importer, "django", batch_size=sample_size, bulk_save=False ) writer.write(to_store) writer.close() @@ -335,7 +335,7 @@ class TestMultipleModels: adapter2 = DjangoModelExportAdapter(model2) exporter.append(adapter1) exporter.append(adapter2) - reader = DjangoBookReader(exporter) + reader = DjangoBookReader(exporter, "django") result = OrderedDict() for index, sheet in enumerate(reader.content_array): result.update( diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 8533e57..e0c8b07 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -441,7 +441,7 @@ class TestMultipleRead: post_adapter.column_names = data["Post"][0] post_adapter.row_initializer = post_init_func importer.append(post_adapter) - writer = SQLBookWriter(importer) + writer = SQLBookWriter(importer, "sql") to_store = OrderedDict() to_store.update({category_adapter.get_name(): data["Category"][1:]}) to_store.update({post_adapter.get_name(): data["Post"][1:]}) @@ -454,7 +454,7 @@ class TestMultipleRead: exporter.append(category_adapter) post_adapter = SQLTableExportAdapter(Post) exporter.append(post_adapter) - reader = SQLBookReader(exporter) + reader = SQLBookReader(exporter, "sql") result = OrderedDict() for index, sheet in enumerate(reader.content_array): result.update( @@ -559,7 +559,7 @@ def test_unknown_sheet(): category_adapter = SQLTableImportAdapter(Category) category_adapter.column_names = [""] importer.append(category_adapter) - writer = SQLBookWriter(importer) + writer = SQLBookWriter(importer, "sql") to_store = OrderedDict() to_store.update({"you do not see me": [[]]}) writer.write(to_store) From d66637afa92b659ac8d237e318c8321d1084f68f Mon Sep 17 00:00:00 2001 From: jaska Date: Mon, 28 Sep 2020 21:44:11 +0100 Subject: [PATCH 088/143] Plugin api (#82) * :tada: abstract reader and writer * :hammer: update sheet interface * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst Co-authored-by: chfw --- pyexcel_io/database/exporters/django.py | 3 +- pyexcel_io/database/exporters/sqlalchemy.py | 3 +- pyexcel_io/database/querysets.py | 15 ++++----- pyexcel_io/io.py | 2 ++ pyexcel_io/plugin_api/__init__.py | 0 pyexcel_io/plugin_api/abstract_reader.py | 9 ++++++ pyexcel_io/plugin_api/abstract_sheet.py | 9 ++++++ pyexcel_io/plugin_api/abstract_writer.py | 9 ++++++ pyexcel_io/readers/csv_in_file.py | 3 +- pyexcel_io/readers/csv_in_memory.py | 3 +- pyexcel_io/readers/csv_sheet.py | 4 ++- pyexcel_io/readers/csvz.py | 3 +- pyexcel_io/writers/csv_in_file.py | 12 ++----- pyexcel_io/writers/csv_in_memory.py | 12 ++----- pyexcel_io/writers/csvz_writer.py | 12 ++----- tests/test_csv_book.py | 24 ++++++-------- tests/test_django_book.py | 35 +++++++++++--------- tests/test_sql_book.py | 36 +++++++++++---------- 18 files changed, 103 insertions(+), 91 deletions(-) create mode 100644 pyexcel_io/plugin_api/__init__.py create mode 100644 pyexcel_io/plugin_api/abstract_reader.py create mode 100644 pyexcel_io/plugin_api/abstract_sheet.py create mode 100644 pyexcel_io/plugin_api/abstract_writer.py diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index 0717cfd..77defa4 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.plugin_api.abstract_reader import IReader class DjangoModelReader(QuerysetsReader): @@ -26,7 +27,7 @@ class DjangoModelReader(QuerysetsReader): ) -class DjangoBookReader(object): +class DjangoBookReader(IReader): """ read django models """ def __init__(self, exporter, _, **keywords): diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index 077cb21..0c31e2e 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.plugin_api.abstract_reader import IReader class SQLTableReader(QuerysetsReader): @@ -30,7 +31,7 @@ class SQLTableReader(QuerysetsReader): QuerysetsReader.__init__(self, everything, column_names, **keywords) -class SQLBookReader(object): +class SQLBookReader(IReader): """ read a table via sqlalchemy """ def __init__(self, exporter, _, **keywords): diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index 61640d4..012427b 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -10,17 +10,19 @@ import datetime from itertools import chain -from pyexcel_io.sheet import SheetReader +from pyexcel_io.plugin_api.abstract_sheet import Sheet -class QuerysetsReader(SheetReader): +class QuerysetsReader(Sheet): """ turn querysets into an array """ - def __init__(self, query_sets, column_names, **keywords): - SheetReader.__init__(self, query_sets, **keywords) + def __init__(self, query_sets, column_names): self.__column_names = column_names self.__query_sets = query_sets + def row_iterator(self): + return chain([self.__column_names], self.__query_sets) + def to_array(self): """ Convert query sets into an array @@ -28,12 +30,9 @@ class QuerysetsReader(SheetReader): if len(self.__query_sets) == 0: yield [] - for element in SheetReader.to_array(self): + for element in Sheet.to_array(self): yield element - def row_iterator(self): - return chain([self.__column_names], self.__query_sets) - def column_iterator(self, row): if self.__column_names is None: return diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 0674a63..84b569d 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -184,6 +184,8 @@ def load_data( reader.open_content(file_content, **keywords) elif file_stream: reader.open_stream(file_stream, **keywords) + else: + raise IOError("Unrecognized options") if sheet_name: result = reader.read_sheet_by_name(sheet_name) elif sheet_index is not None: diff --git a/pyexcel_io/plugin_api/__init__.py b/pyexcel_io/plugin_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyexcel_io/plugin_api/abstract_reader.py b/pyexcel_io/plugin_api/abstract_reader.py new file mode 100644 index 0000000..bd3d1ee --- /dev/null +++ b/pyexcel_io/plugin_api/abstract_reader.py @@ -0,0 +1,9 @@ +from pyexcel_io._compact import OrderedDict + + +class IReader(object): + def read_all(self): + result = OrderedDict() + for index, sheet in enumerate(self.content_array): + result.update({sheet.name: self.read_sheet(index).to_array()}) + return result diff --git a/pyexcel_io/plugin_api/abstract_sheet.py b/pyexcel_io/plugin_api/abstract_sheet.py new file mode 100644 index 0000000..e4fcc01 --- /dev/null +++ b/pyexcel_io/plugin_api/abstract_sheet.py @@ -0,0 +1,9 @@ +class Sheet(object): + def to_array(self): + data = [] + for row in self.row_iterator(): + my_row = [] + for element in self.column_iterator(row): + my_row.append(element) + data.append(my_row) + return data diff --git a/pyexcel_io/plugin_api/abstract_writer.py b/pyexcel_io/plugin_api/abstract_writer.py new file mode 100644 index 0000000..08d7e40 --- /dev/null +++ b/pyexcel_io/plugin_api/abstract_writer.py @@ -0,0 +1,9 @@ +class IWriter(object): + def write(self, incoming_dict): + for sheet_name in incoming_dict: + sheet_writer = self.create_sheet(sheet_name) + if sheet_writer: + sheet_writer.write_array(incoming_dict[sheet_name]) + sheet_writer.close() + else: + raise Exception("Cannot create a sheet writer!") diff --git a/pyexcel_io/readers/csv_in_file.py b/pyexcel_io/readers/csv_in_file.py index 535f8be..9e78b12 100644 --- a/pyexcel_io/readers/csv_in_file.py +++ b/pyexcel_io/readers/csv_in_file.py @@ -5,11 +5,12 @@ import glob from pyexcel_io import constants from pyexcel_io.sheet import NamedContent from pyexcel_io.readers.csv_sheet import CSVFileReader +from pyexcel_io.plugin_api.abstract_reader import IReader DEFAULT_NEWLINE = "\r\n" -class FileReader(object): +class FileReader(IReader): def __init__(self, file_name, file_type, **keywords): """Load content from a file :params str filename: an accessible file path diff --git a/pyexcel_io/readers/csv_in_memory.py b/pyexcel_io/readers/csv_in_memory.py index 3e39521..02fdc04 100644 --- a/pyexcel_io/readers/csv_in_memory.py +++ b/pyexcel_io/readers/csv_in_memory.py @@ -4,11 +4,12 @@ import pyexcel_io._compact as compact from pyexcel_io import constants from pyexcel_io.sheet import NamedContent from pyexcel_io.readers.csv_sheet import CSVinMemoryReader +from pyexcel_io.plugin_api.abstract_reader import IReader DEFAULT_SHEET_SEPARATOR_FORMATTER = f"---{constants.DEFAULT_NAME}---%s" -class MemoryReader(object): +class MemoryReader(IReader): def __init__( self, file_stream, file_type, multiple_sheets=False, **keywords ): diff --git a/pyexcel_io/readers/csv_sheet.py b/pyexcel_io/readers/csv_sheet.py index 9eb72a4..5c58ef4 100644 --- a/pyexcel_io/readers/csv_sheet.py +++ b/pyexcel_io/readers/csv_sheet.py @@ -12,6 +12,7 @@ import csv import pyexcel_io.service as service import pyexcel_io._compact as compact import pyexcel_io.constants as constants +from pyexcel_io.sheet import SheetReader DEFAULT_SEPARATOR = "__" DEFAULT_SHEET_SEPARATOR_FORMATTER = "---%s---" % constants.DEFAULT_NAME + "%s" @@ -91,7 +92,7 @@ class CSVMemoryMapIterator(compact.Iterator): pass -class CSVSheetReader(object): +class CSVSheetReader(SheetReader): """ generic csv file reader""" def __init__( @@ -107,6 +108,7 @@ class CSVSheetReader(object): default_float_nan=None, **keywords ): + SheetReader.__init__(self, sheet, **keywords) self._native_sheet = sheet self._keywords = keywords self._encoding = encoding diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index f435afd..9fdc55e 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -14,9 +14,10 @@ from pyexcel_io import constants from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import StringIO from pyexcel_io.readers.csv_sheet import CSVinMemoryReader +from pyexcel_io.plugin_api.abstract_reader import IReader -class FileReader(object): +class FileReader(IReader): def __init__(self, file_alike_object, file_type, **keywords): self.content_array = [] try: diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py index 111b74c..7a88002 100644 --- a/pyexcel_io/writers/csv_in_file.py +++ b/pyexcel_io/writers/csv_in_file.py @@ -1,8 +1,9 @@ from pyexcel_io import constants from pyexcel_io.writers.csv_sheet import CSVFileWriter +from pyexcel_io.plugin_api.abstract_writer import IWriter -class CsvFileWriter: +class CsvFileWriter(IWriter): def __init__(self, file_alike_object, file_type, **keywords): self._file_alike_object = file_alike_object self._keywords = keywords @@ -21,15 +22,6 @@ class CsvFileWriter: self.__index = self.__index + 1 return self.writer - def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") - def close(self): if self.writer: self.writer.close() diff --git a/pyexcel_io/writers/csv_in_memory.py b/pyexcel_io/writers/csv_in_memory.py index 9e04b9d..ff0e6cb 100644 --- a/pyexcel_io/writers/csv_in_memory.py +++ b/pyexcel_io/writers/csv_in_memory.py @@ -1,8 +1,9 @@ from pyexcel_io import constants from pyexcel_io.writers.csv_sheet import CSVMemoryWriter +from pyexcel_io.plugin_api.abstract_writer import IWriter -class CsvMemoryWriter: +class CsvMemoryWriter(IWriter): def __init__(self, file_alike_object, file_type, **keywords): self._file_alike_object = file_alike_object self._keywords = keywords @@ -21,14 +22,5 @@ class CsvMemoryWriter: self.__index = self.__index + 1 return writer - def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") - def close(self): pass diff --git a/pyexcel_io/writers/csvz_writer.py b/pyexcel_io/writers/csvz_writer.py index ace68c9..0e0379b 100644 --- a/pyexcel_io/writers/csvz_writer.py +++ b/pyexcel_io/writers/csvz_writer.py @@ -2,9 +2,10 @@ import zipfile from pyexcel_io import constants from pyexcel_io.writers.csvz_sheet import CSVZipSheetWriter +from pyexcel_io.plugin_api.abstract_writer import IWriter -class CsvZipWriter(object): +class CsvZipWriter(IWriter): """ csvz writer @@ -29,15 +30,6 @@ class CsvZipWriter(object): ) return writer - def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") - def close(self): if self.zipfile: self.zipfile.close() diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index 381ca69..c23489f 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -4,7 +4,6 @@ from unittest import TestCase import pyexcel_io.manager as manager from pyexcel_io.sheet import NamedContent -from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import BytesIO, StringIO from pyexcel_io.readers.csv_sheet import ( CSVFileReader, @@ -32,9 +31,7 @@ class TestReaders(TestCase): sheet.get_file_handle() def test_sheet_file_reader(self): - r = EncapsulatedSheetReader( - CSVFileReader(NamedContent(self.file_type, self.test_file)) - ) + r = CSVFileReader(NamedContent(self.file_type, self.test_file)) result = list(r.to_array()) self.assertEqual(result, self.expected_data) @@ -43,9 +40,8 @@ class TestReaders(TestCase): with open(self.test_file, "r") as f: io.write(f.read()) io.seek(0) - r = EncapsulatedSheetReader( - CSVinMemoryReader(NamedContent(self.file_type, io)) - ) + r = CSVinMemoryReader(NamedContent(self.file_type, io)) + result = list(r.to_array()) self.assertEqual(result, self.expected_data) @@ -112,9 +108,7 @@ class TestNonUniformCSV(TestCase): f.write(",".join(row) + "\n") def test_sheet_file_reader(self): - r = EncapsulatedSheetReader( - CSVFileReader(NamedContent(self.file_type, self.test_file)) - ) + r = CSVFileReader(NamedContent(self.file_type, self.test_file)) result = list(r.to_array()) self.assertEqual(result, [[1], [4, 5, 6], ["", 7]]) @@ -124,9 +118,8 @@ class TestNonUniformCSV(TestCase): def test_utf16_decoding(): test_file = os.path.join("tests", "fixtures", "csv-encoding-utf16.csv") - reader = EncapsulatedSheetReader( - CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") - ) + reader = CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") + content = list(reader.to_array()) expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] eq_(content, expected) @@ -149,9 +142,10 @@ def test_utf16_encoding(): def test_utf16_memory_decoding(): test_content = u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot" test_content = BytesIO(test_content.encode("utf-16")) - reader = EncapsulatedSheetReader( - CSVinMemoryReader(NamedContent("csv", test_content), encoding="utf-16") + reader = CSVinMemoryReader( + NamedContent("csv", test_content), encoding="utf-16" ) + content = list(reader.to_array()) expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] eq_(content, expected) diff --git a/tests/test_django_book.py b/tests/test_django_book.py index b92356c..fa231c4 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -1,4 +1,5 @@ from pyexcel_io import save_data +from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import OrderedDict from pyexcel_io.constants import DB_DJANGO from pyexcel_io.database.common import ( @@ -210,7 +211,9 @@ class TestSheet: return [str(element) for element in row] # the key point of this test case - reader = DjangoModelReader(model, row_renderer=row_renderer) + reader = EncapsulatedSheetReader( + DjangoModelReader(model), row_renderer=row_renderer + ) data = reader.to_array() expected = [["X", "Y", "Z"], ["1", "2", "3"], ["4", "5", "6"]] eq_(list(data), expected) @@ -336,16 +339,10 @@ class TestMultipleModels: exporter.append(adapter1) exporter.append(adapter2) reader = DjangoBookReader(exporter, "django") - result = OrderedDict() - for index, sheet in enumerate(reader.content_array): - result.update( - { - reader.content_array[index].name: list( - reader.read_sheet(index).to_array() - ) - } - ) - assert result == self.content + result = reader.read_all() + for key in result: + result[key] = list(result[key]) + eq_(result, self.content) @raises(Exception) def test_special_case_where_only_one_model_used(self): @@ -377,25 +374,33 @@ class TestFilter: self.model._meta.update(["X", "Y", "Z"]) def test_load_sheet_from_django_model_with_filter(self): - reader = DjangoModelReader(self.model, start_row=0, row_limit=2) + reader = EncapsulatedSheetReader( + DjangoModelReader(self.model), start_row=0, row_limit=2 + ) data = reader.to_array() expected = [["X", "Y", "Z"], [1, 2, 3]] eq_(list(data), expected) def test_load_sheet_from_django_model_with_filter_1(self): - reader = DjangoModelReader(self.model, start_row=1, row_limit=3) + reader = EncapsulatedSheetReader( + DjangoModelReader(self.model), start_row=1, row_limit=3 + ) data = reader.to_array() expected = [[1, 2, 3], [4, 5, 6]] eq_(list(data), expected) def test_load_sheet_from_django_model_with_filter_2(self): - reader = DjangoModelReader(self.model, start_column=1) + reader = EncapsulatedSheetReader( + DjangoModelReader(self.model), start_column=1 + ) data = reader.to_array() expected = [["Y", "Z"], [2, 3], [5, 6]] eq_(list(data), expected) def test_load_sheet_from_django_model_with_filter_3(self): - reader = DjangoModelReader(self.model, start_column=1, column_limit=1) + reader = EncapsulatedSheetReader( + DjangoModelReader(self.model), start_column=1, column_limit=1 + ) data = reader.to_array() expected = [["Y"], [2], [5]] eq_(list(data), expected) diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index e0c8b07..d37922b 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -14,6 +14,7 @@ from sqlalchemy import ( create_engine, ) from sqlalchemy.orm import backref, relationship, sessionmaker +from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import OrderedDict from pyexcel_io.database.common import ( SQLTableExporter, @@ -124,7 +125,7 @@ class TestSingleRead: ["2014-11-12", 1, "Smith", 12.25], ] # 'pyexcel' here is the table name - assert list(data) == content + eq_(list(data), content) mysession.close() def test_sql_formating(self): @@ -134,8 +135,8 @@ class TestSingleRead: return [str(element) for element in row] # the key for this test case - sheet = SQLTableReader( - mysession, Pyexcel, row_renderer=custom_renderer + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), row_renderer=custom_renderer ) data = sheet.to_array() content = [ @@ -148,7 +149,9 @@ class TestSingleRead: def test_sql_filter(self): mysession = Session() - sheet = SQLTableReader(mysession, Pyexcel, start_row=1) + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), start_row=1 + ) data = sheet.to_array() content = [ ["2014-11-11", 0, "Adam", 11.25], @@ -160,7 +163,9 @@ class TestSingleRead: def test_sql_filter_1(self): mysession = Session() - sheet = SQLTableReader(mysession, Pyexcel, start_row=1, row_limit=1) + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), start_row=1, row_limit=1 + ) data = sheet.to_array() content = [["2014-11-11", 0, "Adam", 11.25]] # 'pyexcel'' here is the table name @@ -169,7 +174,9 @@ class TestSingleRead: def test_sql_filter_2(self): mysession = Session() - sheet = SQLTableReader(mysession, Pyexcel, start_column=1) + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), start_column=1 + ) data = sheet.to_array() content = [ ["id", "name", "weight"], @@ -182,8 +189,8 @@ class TestSingleRead: def test_sql_filter_3(self): mysession = Session() - sheet = SQLTableReader( - mysession, Pyexcel, start_column=1, column_limit=1 + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), start_column=1, column_limit=1 ) data = sheet.to_array() content = [["id"], [0], [1]] @@ -455,15 +462,10 @@ class TestMultipleRead: post_adapter = SQLTableExportAdapter(Post) exporter.append(post_adapter) reader = SQLBookReader(exporter, "sql") - result = OrderedDict() - for index, sheet in enumerate(reader.content_array): - result.update( - { - reader.content_array[index].name: list( - reader.read_sheet(index).to_array() - ) - } - ) + result = reader.read_all() + for key in result: + result[key] = list(result[key]) + assert json.dumps(result) == ( '{"category": [["id", "name"], [1, "News"], [2, "Sports"]], ' + '"post": [["body", "category_id", "id", "pub_date", "title"], ' From 80a2d259d0df872ca86224224dad358e4afbab5a Mon Sep 17 00:00:00 2001 From: jaska Date: Tue, 29 Sep 2020 19:24:16 +0100 Subject: [PATCH 089/143] WIP - Query set reader (#83) * :sparkles: new query reader * :green_heart: add close implementation * :lipstick: update coding style * :bug: update close as class method * :lipstick: use default sheet name * :microscope: test query reader * :lipstick: update coding style * :green_heart: test close method * :books: update change log * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst Co-authored-by: chfw --- CHANGELOG.rst | 2 ++ changelog.yml | 2 ++ pyexcel_io/constants.py | 1 + pyexcel_io/database/__init__.py | 6 +++++- pyexcel_io/database/exporters/queryset.py | 20 ++++++++++++++++++++ pyexcel_io/database/querysets.py | 2 ++ tests/test_sql_book.py | 9 +++++++++ 7 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 pyexcel_io/database/exporters/queryset.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 37abb53..0a57e54 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,8 @@ Change log #. pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted. please read 'Packaging with Pyinstaller' in the documentation. +#. new query set reader plugin. pyexcel<=0.6.4 has used intrusive way of getting + query set source done. it is against the plugin interface. 0.5.20 - 17.7.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index 58ce90b..42f52b2 100644 --- a/changelog.yml +++ b/changelog.yml @@ -12,6 +12,8 @@ releases: details: - pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted. please read 'Packaging with Pyinstaller' in the documentation. + - new query set reader plugin. pyexcel<=0.6.4 has used intrusive way of getting query set + source done. it is against the plugin interface. version: 0.6.0 date: tbd - changes: diff --git a/pyexcel_io/constants.py b/pyexcel_io/constants.py index f969f40..533a196 100644 --- a/pyexcel_io/constants.py +++ b/pyexcel_io/constants.py @@ -49,6 +49,7 @@ FILE_FORMAT_XLSX = "xlsx" FILE_FORMAT_XLSM = "xlsm" DB_SQL = "sql" DB_DJANGO = "django" +DB_QUERYSET = "queryset" KEYWORD_TSV_DIALECT = "excel-tab" KEYWORD_LINE_TERMINATOR = "lineterminator" diff --git a/pyexcel_io/database/__init__.py b/pyexcel_io/database/__init__.py index e5763f7..f28f0d5 100644 --- a/pyexcel_io/database/__init__.py +++ b/pyexcel_io/database/__init__.py @@ -8,9 +8,13 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import NewIOPluginInfoChain -from pyexcel_io.constants import DB_SQL, DB_DJANGO +from pyexcel_io.constants import DB_SQL, DB_DJANGO, DB_QUERYSET NewIOPluginInfoChain(__name__).add_a_reader( + relative_plugin_class_path="exporters.queryset.QueryReader", + locations=["file", "memory", "content"], + file_types=[DB_QUERYSET], +).add_a_reader( relative_plugin_class_path="exporters.django.DjangoBookReader", locations=["file", "memory", "content"], file_types=[DB_DJANGO], diff --git a/pyexcel_io/database/exporters/queryset.py b/pyexcel_io/database/exporters/queryset.py new file mode 100644 index 0000000..fd8ccdb --- /dev/null +++ b/pyexcel_io/database/exporters/queryset.py @@ -0,0 +1,20 @@ +from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.plugin_api.abstract_reader import IReader + + +class QueryReader(IReader): + def __init__(self, query_sets, _, column_names=None, **keywords): + self.query_sets = query_sets + self.column_names = column_names + self.keywords = keywords + self.content_array = [ + QuerysetsReader( + self.query_sets, self.column_names, **self.keywords + ) + ] + + def read_sheet(self, index): + return self.content_array[index] + + def close(self): + pass diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index 012427b..b1555f1 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -10,6 +10,7 @@ import datetime from itertools import chain +from pyexcel_io.constants import DEFAULT_SHEET_NAME from pyexcel_io.plugin_api.abstract_sheet import Sheet @@ -17,6 +18,7 @@ class QuerysetsReader(Sheet): """ turn querysets into an array """ def __init__(self, query_sets, column_names): + self.name = DEFAULT_SHEET_NAME self.__column_names = column_names self.__query_sets = query_sets diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index d37922b..334a781 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -24,6 +24,7 @@ from pyexcel_io.database.common import ( ) from sqlalchemy.ext.declarative import declarative_base from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.database.exporters.queryset import QueryReader from pyexcel_io.database.exporters.sqlalchemy import ( SQLBookReader, SQLTableReader, @@ -226,6 +227,14 @@ class TestSingleWrite: reader = QuerysetsReader(query_sets, self.data[0]) results = reader.to_array() assert list(results) == self.results + + query_sets = mysession.query(Pyexcel).all() + query_reader = QueryReader(query_sets, None, column_names=self.data[0]) + result = query_reader.read_all() + for key in result: + result[key] = list(result[key]) + eq_(result, {"pyexcel_sheet1": self.results}) + query_reader.close() mysession.close() def test_update_existing_row(self): From 13ee5d6c0b8a4278ca301539c44ff49b0bfa1f42 Mon Sep 17 00:00:00 2001 From: jaska Date: Tue, 29 Sep 2020 21:55:01 +0100 Subject: [PATCH 090/143] Code refinement (#84) * :tractor: code refactoring * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :tractor: update code readability * :lipstick: fix a bug * :lipstick: minor update Co-authored-by: chfw --- .github/PULL_REQUEST_TEMPLATE.md | 6 ++---- pyexcel_io/_compact.py | 1 - pyexcel_io/constants.py | 2 +- pyexcel_io/database/__init__.py | 4 ++-- pyexcel_io/database/querysets.py | 6 +++--- pyexcel_io/io.py | 2 +- pyexcel_io/plugin_api/abstract_sheet.py | 2 +- pyexcel_io/plugins.py | 14 +++++++------- pyexcel_io/readers/__init__.py | 4 ++-- pyexcel_io/readers/csv_sheet.py | 11 +++++------ pyexcel_io/writers/__init__.py | 4 ++-- tests/test_csv_book.py | 5 ++++- 12 files changed, 30 insertions(+), 31 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6524ae3..6017f21 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,9 @@ With your PR, here is a check list: -- [ ] Has Test cases written -- [ ] Has all code lines tested +- [ ] Has test cases written? +- [ ] Has all code lines tested? - [ ] Has `make format` been run? - [ ] Please update CHANGELOG.yml(not CHANGELOG.rst) - [ ] Passes all Travis CI builds - [ ] Has fair amount of documentation if your change is complex -- [ ] run 'make format' so as to confirm the pyexcel organisation's coding style -- [ ] Please add yourself to 'contributors' section of pyexcel-io.yml (if not found, please use CONTRIBUTORS.rst) - [ ] Agree on NEW BSD License for your contribution diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index 37a5694..793448f 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -30,7 +30,6 @@ except ImportError: from io import BytesIO, StringIO text_type = str -Iterator = object irange = range diff --git a/pyexcel_io/constants.py b/pyexcel_io/constants.py index 533a196..90f6ab9 100644 --- a/pyexcel_io/constants.py +++ b/pyexcel_io/constants.py @@ -4,7 +4,7 @@ Constants appeared in pyexcel - :copyright: (c) 2014-2019 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License """ # flake8: noqa diff --git a/pyexcel_io/database/__init__.py b/pyexcel_io/database/__init__.py index f28f0d5..98c7df4 100644 --- a/pyexcel_io/database/__init__.py +++ b/pyexcel_io/database/__init__.py @@ -7,10 +7,10 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.plugins import NewIOPluginInfoChain +from pyexcel_io.plugins import IOPluginInfoChainV2 from pyexcel_io.constants import DB_SQL, DB_DJANGO, DB_QUERYSET -NewIOPluginInfoChain(__name__).add_a_reader( +IOPluginInfoChainV2(__name__).add_a_reader( relative_plugin_class_path="exporters.queryset.QueryReader", locations=["file", "memory", "content"], file_types=[DB_QUERYSET], diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index b1555f1..f59cb54 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -11,10 +11,10 @@ import datetime from itertools import chain from pyexcel_io.constants import DEFAULT_SHEET_NAME -from pyexcel_io.plugin_api.abstract_sheet import Sheet +from pyexcel_io.plugin_api.abstract_sheet import ISheet -class QuerysetsReader(Sheet): +class QuerysetsReader(ISheet): """ turn querysets into an array """ def __init__(self, query_sets, column_names): @@ -32,7 +32,7 @@ class QuerysetsReader(Sheet): if len(self.__query_sets) == 0: yield [] - for element in Sheet.to_array(self): + for element in ISheet.to_array(self): yield element def column_iterator(self, row): diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 84b569d..eba9315 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -4,7 +4,7 @@ The io interface to file extensions - :copyright: (c) 2014-2019 by Onni Software Ltd. + :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import os diff --git a/pyexcel_io/plugin_api/abstract_sheet.py b/pyexcel_io/plugin_api/abstract_sheet.py index e4fcc01..1abce6d 100644 --- a/pyexcel_io/plugin_api/abstract_sheet.py +++ b/pyexcel_io/plugin_api/abstract_sheet.py @@ -1,4 +1,4 @@ -class Sheet(object): +class ISheet(object): def to_array(self): data = [] for row in self.row_iterator(): diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 9f6d4d6..6e64758 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -18,9 +18,9 @@ ERROR_MESSAGE_FORMATTER = "one of these plugins for %s data in '%s': %s" UPGRADE_MESSAGE = "Please upgrade the plugin '%s' according to \ plugin compactibility table." READER_PLUGIN = "pyexcel-io reader" -NEW_READER_PLUGIN = "pyexcel-io new reader" +READER_PLUGIN_V2 = "pyexcel-io v2 reader" WRITER_PLUGIN = "pyexcel-io writer" -NEW_WRITER_PLUGIN = "pyexcel-io new writer" +WRITER_PLUGIN_V2 = "pyexcel-io v2 writer" class IOPluginInfo(PluginInfo): @@ -65,7 +65,7 @@ class IOPluginInfoChain(PluginInfoChain): return self.add_a_plugin_instance(a_plugin_info) -class NewIOPluginInfoChain(PluginInfoChain): +class IOPluginInfoChainV2(PluginInfoChain): """provide custom functions to add a reader and a writer """ def add_a_reader( @@ -77,7 +77,7 @@ class NewIOPluginInfoChain(PluginInfoChain): ): """ add pyexcle-io reader plugin info """ a_plugin_info = IOPluginInfo( - NEW_READER_PLUGIN, + READER_PLUGIN_V2, self._get_abs_path(relative_plugin_class_path), file_types=[ f"{location}-{file_type}" @@ -97,7 +97,7 @@ class NewIOPluginInfoChain(PluginInfoChain): ): """ add pyexcle-io writer plugin info """ a_plugin_info = IOPluginInfo( - NEW_WRITER_PLUGIN, + WRITER_PLUGIN_V2, self._get_abs_path(relative_plugin_class_path), file_types=[ f"{location}-{file_type}" @@ -243,8 +243,8 @@ class AllWriters: OLD_READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS) OLD_WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS) -NEW_WRITERS = NewIOManager(NEW_WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS) -NEW_READERS = NewIOManager(NEW_READER_PLUGIN, ioutils.AVAILABLE_READERS) +NEW_WRITERS = NewIOManager(WRITER_PLUGIN_V2, ioutils.AVAILABLE_WRITERS) +NEW_READERS = NewIOManager(READER_PLUGIN_V2, ioutils.AVAILABLE_READERS) READERS = AllReaders() WRITERS = AllWriters() diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index c0be33d..063674c 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -7,9 +7,9 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.plugins import NewIOPluginInfoChain +from pyexcel_io.plugins import IOPluginInfoChainV2 -NewIOPluginInfoChain(__name__).add_a_reader( +IOPluginInfoChainV2(__name__).add_a_reader( relative_plugin_class_path="csv_in_file.FileReader", locations=["file"], file_types=["csv", "tsv"], diff --git a/pyexcel_io/readers/csv_sheet.py b/pyexcel_io/readers/csv_sheet.py index 5c58ef4..06f997e 100644 --- a/pyexcel_io/readers/csv_sheet.py +++ b/pyexcel_io/readers/csv_sheet.py @@ -12,7 +12,7 @@ import csv import pyexcel_io.service as service import pyexcel_io._compact as compact import pyexcel_io.constants as constants -from pyexcel_io.sheet import SheetReader +from pyexcel_io.plugin_api.abstract_sheet import ISheet DEFAULT_SEPARATOR = "__" DEFAULT_SHEET_SEPARATOR_FORMATTER = "---%s---" % constants.DEFAULT_NAME + "%s" @@ -27,7 +27,7 @@ LITTLE_ENDIAN = 0 BIG_ENDIAN = 1 -class CSVMemoryMapIterator(compact.Iterator): +class CSVMemoryMapIterator(object): """ Wrapper class for mmap object @@ -52,7 +52,7 @@ class CSVMemoryMapIterator(compact.Iterator): # \r\x00\x00\x00\n # \x00\x00\x00\x.. self.__zeros_left_in_2_row = 3 - elif encoding == "utf-32-be" or encoding == "utf-16-be": + elif encoding in ["utf-32-be", "utf-16-be"]: self.__zeros_left_in_2_row = 0 self.__endian = BIG_ENDIAN elif encoding == "utf-32-le": @@ -92,7 +92,7 @@ class CSVMemoryMapIterator(compact.Iterator): pass -class CSVSheetReader(SheetReader): +class CSVSheetReader(ISheet): """ generic csv file reader""" def __init__( @@ -108,9 +108,7 @@ class CSVSheetReader(SheetReader): default_float_nan=None, **keywords ): - SheetReader.__init__(self, sheet, **keywords) self._native_sheet = sheet - self._keywords = keywords self._encoding = encoding self.__auto_detect_int = auto_detect_int self.__auto_detect_float = auto_detect_float @@ -120,6 +118,7 @@ class CSVSheetReader(SheetReader): self.__pep_0515_off = pep_0515_off self.__ignore_nan_text = ignore_nan_text self.__default_float_nan = default_float_nan + self._keywords = keywords def get_file_handle(self): """ return me unicde reader for csv """ diff --git a/pyexcel_io/writers/__init__.py b/pyexcel_io/writers/__init__.py index 89c9431..5a8ce4b 100644 --- a/pyexcel_io/writers/__init__.py +++ b/pyexcel_io/writers/__init__.py @@ -7,9 +7,9 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -from pyexcel_io.plugins import NewIOPluginInfoChain +from pyexcel_io.plugins import IOPluginInfoChainV2 -NewIOPluginInfoChain(__name__).add_a_writer( +IOPluginInfoChainV2(__name__).add_a_writer( relative_plugin_class_path="csv_in_file.CsvFileWriter", locations=["file", "content"], file_types=["csv", "tsv"], diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index c23489f..76acfbe 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -4,6 +4,7 @@ from unittest import TestCase import pyexcel_io.manager as manager from pyexcel_io.sheet import NamedContent +from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import BytesIO, StringIO from pyexcel_io.readers.csv_sheet import ( CSVFileReader, @@ -108,7 +109,9 @@ class TestNonUniformCSV(TestCase): f.write(",".join(row) + "\n") def test_sheet_file_reader(self): - r = CSVFileReader(NamedContent(self.file_type, self.test_file)) + r = EncapsulatedSheetReader( + CSVFileReader(NamedContent(self.file_type, self.test_file)) + ) result = list(r.to_array()) self.assertEqual(result, [[1], [4, 5, 6], ["", 7]]) From 27ba6066ba4b3f2d9904b8d0e4fc17147afb000f Mon Sep 17 00:00:00 2001 From: jaska Date: Tue, 29 Sep 2020 22:50:22 +0100 Subject: [PATCH 091/143] Refactor django and sqlalchemy writers (#85) * :fire: code reuse * :tada: update available plugins * :bug: update missing file * :green_heart: update test cases --- pyexcel_io/constants.py | 3 +++ pyexcel_io/database/importers/django.py | 12 ++---------- pyexcel_io/database/importers/sqlalchemy.py | 12 ++---------- pyexcel_io/utils.py | 13 +++++++++++-- tests/test_io.py | 4 ++-- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/pyexcel_io/constants.py b/pyexcel_io/constants.py index 90f6ab9..aa78a30 100644 --- a/pyexcel_io/constants.py +++ b/pyexcel_io/constants.py @@ -47,6 +47,9 @@ FILE_FORMAT_ODS = "ods" FILE_FORMAT_XLS = "xls" FILE_FORMAT_XLSX = "xlsx" FILE_FORMAT_XLSM = "xlsm" +FILE_FORMAT_XLSB = "xlsb" +FILE_FORMAT_HTML = "html" +FILE_FORMAT_PDF = "pdf" DB_SQL = "sql" DB_DJANGO = "django" DB_QUERYSET = "queryset" diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 0a1f5d2..64b0731 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -12,6 +12,7 @@ import logging import pyexcel_io.constants as constants from pyexcel_io.sheet import SheetWriter from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none +from pyexcel_io.plugin_api.abstract_writer import IWriter log = logging.getLogger(__name__) @@ -57,7 +58,7 @@ class DjangoModelWriter(SheetWriter): an_object.save() -class DjangoBookWriter(object): +class DjangoBookWriter(IWriter): """ write data into django models """ def __init__(self, exporter, _, **keywords): @@ -82,14 +83,5 @@ class DjangoBookWriter(object): return sheet_writer - def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") - def close(self): pass diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index c51034f..51b207a 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -10,6 +10,7 @@ import pyexcel_io.constants as constants from pyexcel_io.sheet import SheetWriter from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none +from pyexcel_io.plugin_api.abstract_writer import IWriter class PyexcelSQLSkipRowException(Exception): @@ -71,7 +72,7 @@ class SQLTableWriter(SheetWriter): self._native_book.session.commit() -class SQLBookWriter(object): +class SQLBookWriter(IWriter): """ write data into database tables via sqlalchemy """ def __init__(self, file_content, _, auto_commit=True, **keywords): @@ -93,14 +94,5 @@ class SQLBookWriter(object): return sheet_writer - def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") - def close(self): pass diff --git a/pyexcel_io/utils.py b/pyexcel_io/utils.py index f25f7ab..65477a0 100644 --- a/pyexcel_io/utils.py +++ b/pyexcel_io/utils.py @@ -13,7 +13,13 @@ XLS_PLUGIN = "pyexcel-xls" XLSX_PLUGIN = "pyexcel-xlsx" ODS_PLUGIN = "pyexcel-ods" ODS3_PLUGIN = "pyexcel-ods3" +ODSR_PLUGIN = "pyexcel-odsr" +ODSW_PLUGIN = "pyexcel-odsw" +XLSXR_PLUGIN = "pyexcel-xlsxr" XLSXW_PLUGIN = "pyexcel-xlsxw" +XLSBR_PLUGIN = "pyexcel-xlsbr" +HTMLR_PLUGIN = "pyexcel-htmlr" +PDFR_PLUGIN = "pyexcel-pdfr" IO_ITSELF = "pyexcel-io" AVAILABLE_NEW_READERS = {} @@ -23,17 +29,20 @@ AVAILABLE_READERS = { constants.FILE_FORMAT_XLS: [XLS_PLUGIN], constants.FILE_FORMAT_XLSX: [XLS_PLUGIN, XLSX_PLUGIN], constants.FILE_FORMAT_XLSM: [XLS_PLUGIN, XLSX_PLUGIN], - constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN], + constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN, ODSR_PLUGIN], constants.FILE_FORMAT_TSV: [IO_ITSELF], constants.FILE_FORMAT_CSVZ: [IO_ITSELF], constants.FILE_FORMAT_TSVZ: [IO_ITSELF], + constants.FILE_FORMAT_XLSB: [XLSBR_PLUGIN], + constants.FILE_FORMAT_HTML: [HTMLR_PLUGIN], + constants.FILE_FORMAT_PDF: [PDFR_PLUGIN], } AVAILABLE_WRITERS = { constants.FILE_FORMAT_XLS: [XLS_PLUGIN], constants.FILE_FORMAT_XLSX: [XLSX_PLUGIN, XLSXW_PLUGIN], constants.FILE_FORMAT_XLSM: [XLSX_PLUGIN], - constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN], + constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN, ODSW_PLUGIN], constants.FILE_FORMAT_CSV: [IO_ITSELF], constants.FILE_FORMAT_TSV: [IO_ITSELF], constants.FILE_FORMAT_CSVZ: [IO_ITSELF], diff --git a/tests/test_io.py b/tests/test_io.py index 067754a..e0eec0f 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -75,7 +75,7 @@ def test_wrong_parameter_to_get_writer(): def test_load_ods_data(): msg = "Please install one of these plugins for read data in 'ods': " - msg += "pyexcel-ods,pyexcel-ods3" + msg += "pyexcel-ods,pyexcel-ods3,pyexcel-odsr" try: get_data("test.ods") except exceptions.SupportingPluginAvailableButNotInstalled as e: @@ -85,7 +85,7 @@ def test_load_ods_data(): def test_load_ods_data_from_memory(): io = BytesIO() msg = "Please install one of these plugins for read data in 'ods': " - msg += "pyexcel-ods,pyexcel-ods3" + msg += "pyexcel-ods,pyexcel-ods3,pyexcel-odsr" try: get_data(io, file_type="ods") except exceptions.SupportingPluginAvailableButNotInstalled as e: From efcf49ca14a9ac767e5fc17593faa405f416cc4b Mon Sep 17 00:00:00 2001 From: jaska Date: Wed, 30 Sep 2020 22:23:43 +0100 Subject: [PATCH 092/143] Sheet writer interface (#87) * :fire: code reuse * :tada: update available plugins * :bug: update missing file * :green_heart: update test cases * :sparkles: outline the plugin sheet writer interface. well, break the old interface * :tractor: break the old interface * :fire: remove useless metaphor --- pyexcel_io/database/importers/django.py | 5 +- pyexcel_io/database/importers/sqlalchemy.py | 29 ++++--- pyexcel_io/plugin_api/abstract_sheet.py | 9 +++ pyexcel_io/writers/csv_sheet.py | 83 ++++++++------------- pyexcel_io/writers/csvz_sheet.py | 16 ++-- 5 files changed, 64 insertions(+), 78 deletions(-) diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 64b0731..c79fbe9 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -10,18 +10,17 @@ import logging import pyexcel_io.constants as constants -from pyexcel_io.sheet import SheetWriter from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none +from pyexcel_io.plugin_api.abstract_sheet import ISheetWriter from pyexcel_io.plugin_api.abstract_writer import IWriter log = logging.getLogger(__name__) -class DjangoModelWriter(SheetWriter): +class DjangoModelWriter(ISheetWriter): """ import data into a django model """ def __init__(self, importer, adapter, batch_size=None, bulk_save=True): - SheetWriter.__init__(self, importer, adapter, adapter.name) self.__batch_size = batch_size self.__model = adapter.model self.__column_names = adapter.column_names diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 51b207a..591c838 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -8,8 +8,8 @@ :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants -from pyexcel_io.sheet import SheetWriter from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none +from pyexcel_io.plugin_api.abstract_sheet import ISheetWriter from pyexcel_io.plugin_api.abstract_writer import IWriter @@ -22,18 +22,17 @@ class PyexcelSQLSkipRowException(Exception): pass -class SQLTableWriter(SheetWriter): +class SQLTableWriter(ISheetWriter): """Write to a table""" def __init__( self, importer, adapter, auto_commit=True, bulk_size=1000, **keywords ): - SheetWriter.__init__( - self, importer, adapter, adapter.get_name(), **keywords - ) self.__auto_commit = auto_commit self.__count = 0 self.__bulk_size = bulk_size + self.adapter = adapter + self.importer = importer def write_row(self, array): if is_empty_array(array): @@ -47,29 +46,29 @@ class SQLTableWriter(SheetWriter): print(new_array) def _write_row(self, array): - row = dict(zip(self._native_sheet.column_names, array)) + row = dict(zip(self.adapter.column_names, array)) obj = None - if self._native_sheet.row_initializer: + if self.adapter.row_initializer: # allow initinalizer to return None # if skipping is needed - obj = self._native_sheet.row_initializer(row) + obj = self.adapter.row_initializer(row) if obj is None: - obj = self._native_sheet.table() - for name in self._native_sheet.column_names: - if self._native_sheet.column_name_mapping_dict is not None: - key = self._native_sheet.column_name_mapping_dict[name] + obj = self.adapter.table() + for name in self.adapter.column_names: + if self.adapter.column_name_mapping_dict is not None: + key = self.adapter.column_name_mapping_dict[name] else: key = name setattr(obj, key, row[name]) - self._native_book.session.add(obj) + self.importer.session.add(obj) if self.__auto_commit and self.__bulk_size != float("inf"): self.__count += 1 if self.__count % self.__bulk_size == 0: - self._native_book.session.commit() + self.importer.session.commit() def close(self): if self.__auto_commit: - self._native_book.session.commit() + self.importer.session.commit() class SQLBookWriter(IWriter): diff --git a/pyexcel_io/plugin_api/abstract_sheet.py b/pyexcel_io/plugin_api/abstract_sheet.py index 1abce6d..9121aa7 100644 --- a/pyexcel_io/plugin_api/abstract_sheet.py +++ b/pyexcel_io/plugin_api/abstract_sheet.py @@ -7,3 +7,12 @@ class ISheet(object): my_row.append(element) data.append(my_row) return data + + +class ISheetWriter(object): + def write_array(self, table): + """ + For standalone usage, write an array + """ + for row in table: + self.write_row(row) diff --git a/pyexcel_io/writers/csv_sheet.py b/pyexcel_io/writers/csv_sheet.py index 16c1399..161e808 100644 --- a/pyexcel_io/writers/csv_sheet.py +++ b/pyexcel_io/writers/csv_sheet.py @@ -10,10 +10,10 @@ import csv import pyexcel_io.constants as constants -from pyexcel_io.sheet import SheetWriter +from pyexcel_io.plugin_api.abstract_sheet import ISheetWriter -class CSVSheetWriter(SheetWriter): +class CSVFileWriter(ISheetWriter): """ csv file writer @@ -30,20 +30,39 @@ class CSVSheetWriter(SheetWriter): ): self._encoding = encoding self._sheet_name = name + if self._sheet_name is None or single_sheet_in_book: + self._sheet_name = constants.DEFAULT_SHEET_NAME self._single_sheet_in_book = single_sheet_in_book self.__line_terminator = constants.DEFAULT_CSV_NEWLINE + self._keywords = keywords if constants.KEYWORD_LINE_TERMINATOR in keywords: self.__line_terminator = keywords.get( constants.KEYWORD_LINE_TERMINATOR ) - if single_sheet_in_book: - self._sheet_name = None self._sheet_index = sheet_index - self.writer = None self.file_handle = None - SheetWriter.__init__( - self, filename, self._sheet_name, self._sheet_name, **keywords + self._native_book = filename + + self.writer = self.get_writer() + + def get_writer(self): + if self._sheet_name != constants.DEFAULT_SHEET_NAME: + names = self._native_book.split(".") + file_name = "%s%s%s%s%s.%s" % ( + names[0], + constants.DEFAULT_MULTI_CSV_SEPARATOR, + self._sheet_name, # sheet name + constants.DEFAULT_MULTI_CSV_SEPARATOR, + self._sheet_index, # sheet index + names[1], + ) + else: + file_name = self._native_book + + self.file_handle = open( + file_name, "w", newline="", encoding=self._encoding ) + return csv.writer(self.file_handle, **self._keywords) def write_row(self, array): """ @@ -51,64 +70,24 @@ class CSVSheetWriter(SheetWriter): """ self.writer.writerow(array) - -class CSVFileWriter(CSVSheetWriter): - """ Write csv to a physical file """ - def close(self): self.file_handle.close() - def set_sheet_name(self, name): - if name != constants.DEFAULT_SHEET_NAME: - names = self._native_book.split(".") - file_name = "%s%s%s%s%s.%s" % ( - names[0], - constants.DEFAULT_MULTI_CSV_SEPARATOR, - name, # sheet name - constants.DEFAULT_MULTI_CSV_SEPARATOR, - self._sheet_index, # sheet index - names[1], - ) - else: - file_name = self._native_book - self.file_handle = open( - file_name, "w", newline="", encoding=self._encoding - ) - self.writer = csv.writer(self.file_handle, **self._keywords) - -class CSVMemoryWriter(CSVSheetWriter): +class CSVMemoryWriter(CSVFileWriter): """ Write csv to a memory stream """ - def __init__( - self, - filename, - name, - encoding="utf-8", - single_sheet_in_book=False, - sheet_index=None, - **keywords - ): - CSVSheetWriter.__init__( - self, - filename, - name, - encoding=encoding, - single_sheet_in_book=single_sheet_in_book, - sheet_index=sheet_index, - **keywords - ) - - def set_sheet_name(self, name): + def get_writer(self): self.file_handle = self._native_book - self.writer = csv.writer(self.file_handle, **self._keywords) + writer = csv.writer(self.file_handle, **self._keywords) if not self._single_sheet_in_book: - self.writer.writerow( + writer.writerow( [ constants.DEFAULT_CSV_STREAM_FILE_FORMATTER % (self._sheet_name, "") ] ) + return writer def close(self): if self._single_sheet_in_book: diff --git a/pyexcel_io/writers/csvz_sheet.py b/pyexcel_io/writers/csvz_sheet.py index f4b0ee9..b674449 100644 --- a/pyexcel_io/writers/csvz_sheet.py +++ b/pyexcel_io/writers/csvz_sheet.py @@ -7,26 +7,26 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +import csv + from pyexcel_io._compact import StringIO -from pyexcel_io.writers.csv_sheet import CSVSheetWriter +from pyexcel_io.writers.csv_sheet import CSVFileWriter -class CSVZipSheetWriter(CSVSheetWriter): +class CSVZipSheetWriter(CSVFileWriter): """ handle the zipfile interface """ def __init__(self, zipfile, sheetname, file_extension, **keywords): self.file_extension = file_extension keywords["single_sheet_in_book"] = False - CSVSheetWriter.__init__(self, zipfile, sheetname, **keywords) - - def set_sheet_name(self, name): self.content = StringIO() - import csv + super().__init__(zipfile, sheetname, **keywords) - self.writer = csv.writer(self.content, **self._keywords) + def get_writer(self): + return csv.writer(self.content, **self._keywords) def close(self): - file_name = "%s.%s" % (self._native_sheet, self.file_extension) + file_name = "%s.%s" % (self._sheet_name, self.file_extension) self.content.seek(0) self._native_book.writestr(file_name, self.content.read()) self.content.close() From dacbbf33f3520c8077e2f1be7180f338126433b5 Mon Sep 17 00:00:00 2001 From: jaska Date: Wed, 30 Sep 2020 22:45:04 +0100 Subject: [PATCH 093/143] =?UTF-8?q?allow=20trailing=20options,=20get=5Fdat?= =?UTF-8?q?a(...keep=5Ftrailing=5Fempty=5Fcells=3D=E2=80=A6=20(#88)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :tada: allow trailing options, get_data(...keep_trailing_empty_cells=True). fix #86 * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :lipstick: update coding style Co-authored-by: chfw --- CHANGELOG.rst | 5 +++++ changelog.yml | 3 +++ docs/source/index.rst | 1 + docs/source/options.rst | 11 +++++++++++ pyexcel_io/reader.py | 1 + pyexcel_io/sheet.py | 13 +++++++++---- tests/test_csv_book.py | 15 +++++++++++++++ 7 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 docs/source/options.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0a57e54..53e473e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,11 @@ Change log 0.6.0 - tbd -------------------------------------------------------------------------------- +**added** + +#. `#86 `_: allow trailing + options, get_data(...keep_trailing_empty_cells=True). + **fixed** #. `#74 `_: handle zip files diff --git a/changelog.yml b/changelog.yml index 42f52b2..ea6d4bd 100644 --- a/changelog.yml +++ b/changelog.yml @@ -2,6 +2,9 @@ name: pyexcel-io organisation: pyexcel releases: - changes: + - action: added + details: + - "`#86`: allow trailing options, get_data(...keep_trailing_empty_cells=True)." - action: fixed details: - "`#74`: handle zip files which contain non-UTF-8 encoded files." diff --git a/docs/source/index.rst b/docs/source/index.rst index bd24745..49f216e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -91,6 +91,7 @@ get_data(.., library='pyexcel-ods') csvz sqlalchemy django + options extensions diff --git a/docs/source/options.rst b/docs/source/options.rst new file mode 100644 index 0000000..9ef9db4 --- /dev/null +++ b/docs/source/options.rst @@ -0,0 +1,11 @@ +Options +====================== + +Here is the documentation on the keyword options for get_data. + +keep_trailing_empty_cells +------------------------------ + +default: False + +If turned on, the return data will contain trailing empty cells. diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py index 49d08c3..0a1a5ea 100644 --- a/pyexcel_io/reader.py +++ b/pyexcel_io/reader.py @@ -17,6 +17,7 @@ def clean_keywords(keywords): "skip_row_func", "skip_empty_rows", "row_renderer", + "keep_trailing_empty_cells", ] for arg in keywords: if arg in args_list: diff --git a/pyexcel_io/sheet.py b/pyexcel_io/sheet.py index 5afd982..b23f342 100644 --- a/pyexcel_io/sheet.py +++ b/pyexcel_io/sheet.py @@ -38,6 +38,7 @@ class SheetReader(object): skip_column_func=None, skip_empty_rows=False, row_renderer=None, + keep_trailing_empty_cells=False, **deprecated_use_of_keywords_here ): self._native_sheet = sheet @@ -51,6 +52,7 @@ class SheetReader(object): self._skip_column = _index_filter self._skip_empty_rows = skip_empty_rows self._row_renderer = row_renderer + self.keep_trailing_empty_cells = keep_trailing_empty_cells if skip_row_func: self._skip_row = skip_row_func @@ -84,10 +86,13 @@ class SheetReader(object): elif column_position == constants.STOP_ITERATION: break - tmp_row.append(cell_value) - if cell_value is not None and cell_value != "": - return_row += tmp_row - tmp_row = [] + if self.keep_trailing_empty_cells: + return_row.append(cell_value) + else: + tmp_row.append(cell_value) + if cell_value is not None and cell_value != "": + return_row += tmp_row + tmp_row = [] if self._skip_empty_rows and len(return_row) < 1: # we by-pass next yeild here # because it is an empty row diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index 76acfbe..9b64915 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -3,6 +3,7 @@ from textwrap import dedent from unittest import TestCase import pyexcel_io.manager as manager +from pyexcel_io import get_data from pyexcel_io.sheet import NamedContent from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import BytesIO, StringIO @@ -115,6 +116,20 @@ class TestNonUniformCSV(TestCase): result = list(r.to_array()) self.assertEqual(result, [[1], [4, 5, 6], ["", 7]]) + def test_sheet_file_reader_with_trailing_empty_cells(self): + r = EncapsulatedSheetReader( + CSVFileReader(NamedContent(self.file_type, self.test_file)), + keep_trailing_empty_cells=True, + ) + result = list(r.to_array()) + self.assertEqual(result, [[1], [4, 5, 6, "", ""], ["", 7]]) + + def test_get_data_with_trailing_empty_cells(self): + result = get_data(self.test_file, keep_trailing_empty_cells=True) + self.assertEqual( + result[self.test_file], [[1], [4, 5, 6, "", ""], ["", 7]] + ) + def tearDown(self): os.unlink(self.test_file) From 29c26680a753d49eeccf65252dda43a24190ddf9 Mon Sep 17 00:00:00 2001 From: jaska Date: Thu, 1 Oct 2020 21:51:55 +0100 Subject: [PATCH 094/143] Integration with pyexcel-xlsx (#89) * :lipstick: minor tweak in code * :green_heart: use value error --- pyexcel_io/plugins.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 6e64758..7e6625a 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -130,7 +130,10 @@ class IOManager(PluginManager): def get_a_plugin(self, file_type=None, library=None, **keywords): __file_type = file_type.lower() - plugin = self.load_me_now(__file_type, library=library) + try: + plugin = self.load_me_now(__file_type, library=library) + except Exception: + self.raise_exception(__file_type) handler = plugin() handler.set_type(__file_type) return handler From fa808870d3019fbb987cfde0332a6c224f11e4e2 Mon Sep 17 00:00:00 2001 From: jaska Date: Sun, 4 Oct 2020 22:15:23 +0100 Subject: [PATCH 095/143] Improve reader interface (#90) * :hammer: improve reader interface * :hammer: shrink reader code * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :fire: remove redundant functionalitoes, never will use. what's the point * :books: updated doc string and the tutorial * :hammer: update import statements * :microscope: more test coverage * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :green_heart: fix unit test failure * :books: update reader plugin example * :lipstick: update coding style * :books: fix index rst file * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst Co-authored-by: chfw --- .../docs/source/{index.rst => index.rst.jj2} | 0 .moban.yml | 2 +- CONTRIBUTORS.rst | 1 + docs/source/extensions.rst | 28 +++++ docs/source/index.rst | 109 +++++++++++++++++- examples/custom_yaml_reader.py | 45 ++++++++ examples/test.yaml | 11 ++ pyexcel-io.yml | 2 +- pyexcel_io/_compact.py | 4 - pyexcel_io/database/exporters/django.py | 2 +- pyexcel_io/database/exporters/queryset.py | 2 +- pyexcel_io/database/exporters/sqlalchemy.py | 2 +- pyexcel_io/database/importers/django.py | 3 +- pyexcel_io/database/importers/sqlalchemy.py | 3 +- pyexcel_io/database/querysets.py | 7 +- pyexcel_io/plugin_api/__init__.py | 3 + pyexcel_io/plugin_api/abstract_reader.py | 21 +++- pyexcel_io/plugin_api/abstract_sheet.py | 16 +-- pyexcel_io/plugin_api/abstract_writer.py | 6 + pyexcel_io/reader.py | 33 ++---- pyexcel_io/readers/csv_in_file.py | 2 +- pyexcel_io/readers/csv_in_memory.py | 2 +- pyexcel_io/readers/csv_sheet.py | 4 +- pyexcel_io/writers/csv_in_file.py | 2 +- pyexcel_io/writers/csv_in_memory.py | 2 +- pyexcel_io/writers/csv_sheet.py | 4 +- pyexcel_io/writers/csvz_sheet.py | 4 +- pyexcel_io/writers/csvz_writer.py | 2 +- tests/requirements.txt | 2 +- tests/test_csv_book.py | 16 ++- tests/test_django_book.py | 9 +- tests/test_plugin_api.py | 53 +++++++++ tests/test_sql_book.py | 11 +- 33 files changed, 337 insertions(+), 76 deletions(-) rename .moban.d/docs/source/{index.rst => index.rst.jj2} (100%) create mode 100644 examples/custom_yaml_reader.py create mode 100644 examples/test.yaml create mode 100644 tests/test_plugin_api.py diff --git a/.moban.d/docs/source/index.rst b/.moban.d/docs/source/index.rst.jj2 similarity index 100% rename from .moban.d/docs/source/index.rst rename to .moban.d/docs/source/index.rst.jj2 diff --git a/.moban.yml b/.moban.yml index 98dbc6f..b73e1fa 100644 --- a/.moban.yml +++ b/.moban.yml @@ -6,5 +6,5 @@ targets: - setup.py: io_setup.py.jj2 - .travis.yml: custom_travis.yml.jj2 - README.rst: io_readme.rst.jj2 - - "docs/source/index.rst": "docs/source/index.rst" + - "docs/source/index.rst": "docs/source/index.rst.jj2" - .gitignore: gitignore.jj2 diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 6d0fa16..ba2e01d 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -1,4 +1,5 @@ + 5 contributors ================================================================================ diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index 20456d2..a9b6c68 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -1,3 +1,31 @@ +Extend pyexcel-io Tutorial +================================================================================ + +pyexcel-io itself comes with csv support. + +Reader +-------------------------------------------------------------------------------- + +Suppose we have a yaml file, containing a dictionary where the values are +two dimensional array. The task is write reader plugin to pyexcel-io so that +we can use get_data() to read it out. + +Example yaml data:: + +.. literalinclude:: ../../examples/test.yaml + :language: yaml + +Example code:: + +.. literalinclude:: ../../examples/custom_yeaml_reader.py + :language: python + + +Writer +-------------------------------------------------------------------------------- + + + Working with xls, xlsx, and ods formats ================================================================================ diff --git a/docs/source/index.rst b/docs/source/index.rst index 49f216e..db51486 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,7 +3,16 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -{%include "header.rst.jj2" %} +`pyexcel-io` - Let you focus on data, instead of file formats +================================================================================ + +:Author: chfw +:Source code: http://github.com/pyexcel/pyexcel-io.git +:Issues: http://github.com/pyexcel/pyexcel-io/issues +:License: New BSD License +:Development: |release| +:Released: |version| +:Generated: |today| Introduction -------------------------------------------------------------------------------- @@ -33,11 +42,104 @@ as of 2014. They are invented and supported by `pyexcel-io`_. Installation -------------------------------------------------------------------------------- -{%include "installation.rst.jj2" %} + +You can install pyexcel-io via pip: + +.. code-block:: bash + + $ pip install pyexcel-io + + +or clone it and install it: + +.. code-block:: bash + + $ git clone https://github.com/pyexcel/pyexcel-io.git + $ cd pyexcel-io + $ python setup.py install For individual excel file formats, please install them as you wish: -{%include "io-plugins-list.rst.jj2" %} +.. _file-format-list: +.. _a-map-of-plugins-and-file-formats: + +.. table:: A list of file formats supported by external plugins + + ======================== ======================= ================= ================== + Package name Supported file formats Dependencies Python versions + ======================== ======================= ================= ================== + `pyexcel-io`_ >=v0.6.0 csv, csvz [#f1]_, tsv, 3.6+ + tsvz [#f2]_ + `pyexcel-io`_ <=0.5.20 same as above 2.6, 2.7, 3.3, + 3.4, 3.5, 3.6 + pypy + `pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, same as above + xlsm(read only) `xlwt`_ + `pyexcel-xlsx`_ xlsx `openpyxl`_ same as above + `pyexcel-ods3`_ ods `pyexcel-ezodf`_, 2.6, 2.7, 3.3, 3.4 + lxml 3.5, 3.6 + `pyexcel-ods`_ ods `odfpy`_ same as above + ======================== ======================= ================= ================== + +.. table:: Dedicated file reader and writers + + ======================== ======================= ================= ================== + Package name Supported file formats Dependencies Python versions + ======================== ======================= ================= ================== + `pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ Python 2 and 3 + `pyexcel-xlsxr`_ xlsx(read only) lxml same as above + `pyexcel-xlsbr`_ xlsx(read only) pyxlsb same as above + `pyexcel-odsr`_ read only for ods, fods lxml same as above + `pyexcel-odsw`_ write only for ods loxun same as above + `pyexcel-htmlr`_ html(read only) lxml,html5lib same as above + `pyexcel-pdfr`_ pdf(read only) pdftables Python 2 only. + ======================== ======================= ================= ================== + + +Plugin shopping guide +------------------------ + +Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of +xml files + +The dedicated readers for excel files can stream read + + +In order to manage the list of plugins installed, you need to use pip to add or remove +a plugin. When you use virtualenv, you can have different plugins per virtual +environment. In the situation where you have multiple plugins that does the same thing +in your environment, you need to tell pyexcel which plugin to use per function call. +For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr. +You need to append get_array(..., library='pyexcel-odsr'). + + + +.. _pyexcel-io: https://github.com/pyexcel/pyexcel-io +.. _pyexcel-xls: https://github.com/pyexcel/pyexcel-xls +.. _pyexcel-xlsx: https://github.com/pyexcel/pyexcel-xlsx +.. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods +.. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3 +.. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr +.. _pyexcel-odsw: https://github.com/pyexcel/pyexcel-odsw +.. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr + +.. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw +.. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr +.. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr +.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr + +.. _xlrd: https://github.com/python-excel/xlrd +.. _xlwt: https://github.com/python-excel/xlwt +.. _openpyxl: https://bitbucket.org/openpyxl/openpyxl +.. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter +.. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf +.. _odfpy: https://github.com/eea/odfpy + + +.. rubric:: Footnotes + +.. [#f1] zipped csv file +.. [#f2] zipped tsv file After that, you can start get and save data in the loaded format. There are two plugins for the same file format, e.g. pyexcel-ods3 and pyexcel-ods. @@ -91,7 +193,6 @@ get_data(.., library='pyexcel-ods') csvz sqlalchemy django - options extensions diff --git a/examples/custom_yaml_reader.py b/examples/custom_yaml_reader.py new file mode 100644 index 0000000..ef68606 --- /dev/null +++ b/examples/custom_yaml_reader.py @@ -0,0 +1,45 @@ +import yaml +from pyexcel_io import get_data +from pyexcel_io.sheet import NamedContent +from pyexcel_io.plugins import IOPluginInfoChainV2 +from pyexcel_io.plugin_api import ISheet, IReader + + +class YourSingleSheet(ISheet): + def __init__(self, your_native_sheet): + self.two_dimensional_array = your_native_sheet + + def row_iterator(self): + yield from self.two_dimensional_array + + def column_iterator(self, row): + yield from row + + +class YourReader(IReader): + def __init__(self, file_name, file_type, **keywords): + self.file_handle = open(file_name, "r") + self.native_book = yaml.load(self.file_handle) + self.content_array = [ + NamedContent(key, values) + for key, values in self.native_book.items() + ] + + def read_sheet(self, sheet_index): + two_dimensional_array = self.content_array[sheet_index].payload + return YourSingleSheet(two_dimensional_array) + + def close(self): + self.file_handle.close() + + +IOPluginInfoChainV2(__name__).add_a_reader( + relative_plugin_class_path="YourReader", + locations=["file"], + file_types=["yaml"], + stream_type="text", +) + +if __name__ == "__main__": + data = get_data("test.yaml") + print(data) diff --git a/examples/test.yaml b/examples/test.yaml new file mode 100644 index 0000000..1e02d55 --- /dev/null +++ b/examples/test.yaml @@ -0,0 +1,11 @@ +sheet 1: +- - 1 + - 2 + - 3 +- - 2 + - 3 + - 4 +sheet 2: +- - A + - B + - C diff --git a/pyexcel-io.yml b/pyexcel-io.yml index b7818fa..ecf687a 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -13,7 +13,7 @@ dependencies: - lml>=0.0.4 test_dependencies: - pyexcel - - pyexcel-xls + - pyexcel-xls==0.5.9 - SQLAlchemy - pyexcel-xlsxw extra_dependencies: diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index 793448f..6b691c4 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -51,8 +51,4 @@ def is_string(atype): if atype == str: return True - elif PY2: - if atype == unicode: - return True - return False diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index 77defa4..d642c5f 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -7,8 +7,8 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +from pyexcel_io.plugin_api import IReader from pyexcel_io.database.querysets import QuerysetsReader -from pyexcel_io.plugin_api.abstract_reader import IReader class DjangoModelReader(QuerysetsReader): diff --git a/pyexcel_io/database/exporters/queryset.py b/pyexcel_io/database/exporters/queryset.py index fd8ccdb..6cca0e7 100644 --- a/pyexcel_io/database/exporters/queryset.py +++ b/pyexcel_io/database/exporters/queryset.py @@ -1,5 +1,5 @@ +from pyexcel_io.plugin_api import IReader from pyexcel_io.database.querysets import QuerysetsReader -from pyexcel_io.plugin_api.abstract_reader import IReader class QueryReader(IReader): diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index 0c31e2e..1e084b8 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -7,8 +7,8 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +from pyexcel_io.plugin_api import IReader from pyexcel_io.database.querysets import QuerysetsReader -from pyexcel_io.plugin_api.abstract_reader import IReader class SQLTableReader(QuerysetsReader): diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index c79fbe9..9d7307e 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -11,8 +11,7 @@ import logging import pyexcel_io.constants as constants from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none -from pyexcel_io.plugin_api.abstract_sheet import ISheetWriter -from pyexcel_io.plugin_api.abstract_writer import IWriter +from pyexcel_io.plugin_api import IWriter, ISheetWriter log = logging.getLogger(__name__) diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 591c838..5c5cd27 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -9,8 +9,7 @@ """ import pyexcel_io.constants as constants from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none -from pyexcel_io.plugin_api.abstract_sheet import ISheetWriter -from pyexcel_io.plugin_api.abstract_writer import IWriter +from pyexcel_io.plugin_api import IWriter, ISheetWriter class PyexcelSQLSkipRowException(Exception): diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index f59cb54..d30ae05 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -32,8 +32,11 @@ class QuerysetsReader(ISheet): if len(self.__query_sets) == 0: yield [] - for element in ISheet.to_array(self): - yield element + for row in self.row_iterator(): + row_values = [] + for value in self.column_iterator(row): + row_values.append(value) + yield row_values def column_iterator(self, row): if self.__column_names is None: diff --git a/pyexcel_io/plugin_api/__init__.py b/pyexcel_io/plugin_api/__init__.py index e69de29..bb39d55 100644 --- a/pyexcel_io/plugin_api/__init__.py +++ b/pyexcel_io/plugin_api/__init__.py @@ -0,0 +1,3 @@ +from .abstract_sheet import ISheet, ISheetWriter # noqa: F401 +from .abstract_reader import IReader # noqa: F401 +from .abstract_writer import IWriter # noqa: F401 diff --git a/pyexcel_io/plugin_api/abstract_reader.py b/pyexcel_io/plugin_api/abstract_reader.py index bd3d1ee..12dc919 100644 --- a/pyexcel_io/plugin_api/abstract_reader.py +++ b/pyexcel_io/plugin_api/abstract_reader.py @@ -1,9 +1,18 @@ -from pyexcel_io._compact import OrderedDict +from .abstract_sheet import ISheet class IReader(object): - def read_all(self): - result = OrderedDict() - for index, sheet in enumerate(self.content_array): - result.update({sheet.name: self.read_sheet(index).to_array()}) - return result + """ + content_array should be a list of NamedContent + where: name is the sheet name, + payload is the native sheet. + """ + + def read_sheet(self, sheet_index) -> ISheet: + raise NotImplementedError("") + + def sheet_names(self): + return [content.name for content in self.content_array] + + def __len__(self): + return len(self.content_array) diff --git a/pyexcel_io/plugin_api/abstract_sheet.py b/pyexcel_io/plugin_api/abstract_sheet.py index 9121aa7..7e82aa7 100644 --- a/pyexcel_io/plugin_api/abstract_sheet.py +++ b/pyexcel_io/plugin_api/abstract_sheet.py @@ -1,15 +1,15 @@ class ISheet(object): - def to_array(self): - data = [] - for row in self.row_iterator(): - my_row = [] - for element in self.column_iterator(row): - my_row.append(element) - data.append(my_row) - return data + def row_iterator(self): + raise NotImplementedError("") + + def column_iterator(self, row): + raise NotImplementedError("") class ISheetWriter(object): + def write_row(self, data_row): + raise NotImplementedError("How does your sheet write a row of data") + def write_array(self, table): """ For standalone usage, write an array diff --git a/pyexcel_io/plugin_api/abstract_writer.py b/pyexcel_io/plugin_api/abstract_writer.py index 08d7e40..0f0c8f5 100644 --- a/pyexcel_io/plugin_api/abstract_writer.py +++ b/pyexcel_io/plugin_api/abstract_writer.py @@ -1,4 +1,10 @@ +from .abstract_sheet import ISheetWriter + + class IWriter(object): + def create_sheet(self, sheet_name) -> ISheetWriter: + raise NotImplementedError("Please implement a native sheet writer") + def write(self, incoming_dict): for sheet_name in incoming_dict: sheet_writer = self.create_sheet(sheet_name) diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py index 0a1a5ea..704a709 100644 --- a/pyexcel_io/reader.py +++ b/pyexcel_io/reader.py @@ -76,38 +76,25 @@ class Reader(object): """ read a named sheet from a excel data book """ - for index, content in enumerate(self.reader.content_array): - if content.name == sheet_name: - return {content.name: self.read_sheet(index)} - else: - raise ValueError("Cannot find sheet %s" % sheet_name) + sheet_names = self.reader.sheet_names() + index = sheet_names.index(sheet_name) - def read_sheet(self, sheet_index): - sheet_reader = self.reader.read_sheet(sheet_index) - sheet = EncapsulatedSheetReader(sheet_reader, **self.keywords) - return sheet.to_array() + return self.read_sheet_by_index(index) def read_sheet_by_index(self, sheet_index): - """ - read an indexed sheet from a excel data book - """ - try: - name = self.reader.content_array[sheet_index].name - return {name: self.read_sheet(sheet_index)} - - except IndexError: - self.close() - raise + sheet_reader = self.reader.read_sheet(sheet_index) + sheet_names = self.reader.sheet_names() + sheet = EncapsulatedSheetReader(sheet_reader, **self.keywords) + return {sheet_names[sheet_index]: sheet.to_array()} def read_all(self): """ read everything from a excel data book """ result = OrderedDict() - for index, sheet in enumerate(self.reader.content_array): - result.update( - {self.reader.content_array[index].name: self.read_sheet(index)} - ) + for sheet_index in range(len(self.reader)): + content_dict = self.read_sheet_by_index(sheet_index) + result.update(content_dict) return result def read_many(self, sheets): diff --git a/pyexcel_io/readers/csv_in_file.py b/pyexcel_io/readers/csv_in_file.py index 9e78b12..8e76930 100644 --- a/pyexcel_io/readers/csv_in_file.py +++ b/pyexcel_io/readers/csv_in_file.py @@ -4,8 +4,8 @@ import glob from pyexcel_io import constants from pyexcel_io.sheet import NamedContent +from pyexcel_io.plugin_api import IReader from pyexcel_io.readers.csv_sheet import CSVFileReader -from pyexcel_io.plugin_api.abstract_reader import IReader DEFAULT_NEWLINE = "\r\n" diff --git a/pyexcel_io/readers/csv_in_memory.py b/pyexcel_io/readers/csv_in_memory.py index 02fdc04..eb8ba1d 100644 --- a/pyexcel_io/readers/csv_in_memory.py +++ b/pyexcel_io/readers/csv_in_memory.py @@ -3,8 +3,8 @@ import re import pyexcel_io._compact as compact from pyexcel_io import constants from pyexcel_io.sheet import NamedContent +from pyexcel_io.plugin_api import IReader from pyexcel_io.readers.csv_sheet import CSVinMemoryReader -from pyexcel_io.plugin_api.abstract_reader import IReader DEFAULT_SHEET_SEPARATOR_FORMATTER = f"---{constants.DEFAULT_NAME}---%s" diff --git a/pyexcel_io/readers/csv_sheet.py b/pyexcel_io/readers/csv_sheet.py index 06f997e..8ab6608 100644 --- a/pyexcel_io/readers/csv_sheet.py +++ b/pyexcel_io/readers/csv_sheet.py @@ -1,5 +1,5 @@ """ - pyexcel_io.readers.csvr + pyexcel_io.readers.csv_sheet ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ csv file reader @@ -12,7 +12,7 @@ import csv import pyexcel_io.service as service import pyexcel_io._compact as compact import pyexcel_io.constants as constants -from pyexcel_io.plugin_api.abstract_sheet import ISheet +from pyexcel_io.plugin_api import ISheet DEFAULT_SEPARATOR = "__" DEFAULT_SHEET_SEPARATOR_FORMATTER = "---%s---" % constants.DEFAULT_NAME + "%s" diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py index 7a88002..21eb85c 100644 --- a/pyexcel_io/writers/csv_in_file.py +++ b/pyexcel_io/writers/csv_in_file.py @@ -1,6 +1,6 @@ from pyexcel_io import constants +from pyexcel_io.plugin_api import IWriter from pyexcel_io.writers.csv_sheet import CSVFileWriter -from pyexcel_io.plugin_api.abstract_writer import IWriter class CsvFileWriter(IWriter): diff --git a/pyexcel_io/writers/csv_in_memory.py b/pyexcel_io/writers/csv_in_memory.py index ff0e6cb..e768f4c 100644 --- a/pyexcel_io/writers/csv_in_memory.py +++ b/pyexcel_io/writers/csv_in_memory.py @@ -1,6 +1,6 @@ from pyexcel_io import constants +from pyexcel_io.plugin_api import IWriter from pyexcel_io.writers.csv_sheet import CSVMemoryWriter -from pyexcel_io.plugin_api.abstract_writer import IWriter class CsvMemoryWriter(IWriter): diff --git a/pyexcel_io/writers/csv_sheet.py b/pyexcel_io/writers/csv_sheet.py index 161e808..3c2185f 100644 --- a/pyexcel_io/writers/csv_sheet.py +++ b/pyexcel_io/writers/csv_sheet.py @@ -1,5 +1,5 @@ """ - pyexcel_io.writers.csvw + pyexcel_io.writers.csv_sheet ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The lower level csv file format writer @@ -10,7 +10,7 @@ import csv import pyexcel_io.constants as constants -from pyexcel_io.plugin_api.abstract_sheet import ISheetWriter +from pyexcel_io.plugin_api import ISheetWriter class CSVFileWriter(ISheetWriter): diff --git a/pyexcel_io/writers/csvz_sheet.py b/pyexcel_io/writers/csvz_sheet.py index b674449..26685cd 100644 --- a/pyexcel_io/writers/csvz_sheet.py +++ b/pyexcel_io/writers/csvz_sheet.py @@ -1,6 +1,6 @@ """ - pyexcel_io.fileformat.csvz - ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + pyexcel_io.fileformat.csvz_sheet + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The lower level csvz file format handler. diff --git a/pyexcel_io/writers/csvz_writer.py b/pyexcel_io/writers/csvz_writer.py index 0e0379b..7461ecc 100644 --- a/pyexcel_io/writers/csvz_writer.py +++ b/pyexcel_io/writers/csvz_writer.py @@ -1,8 +1,8 @@ import zipfile from pyexcel_io import constants +from pyexcel_io.plugin_api import IWriter from pyexcel_io.writers.csvz_sheet import CSVZipSheetWriter -from pyexcel_io.plugin_api.abstract_writer import IWriter class CsvZipWriter(IWriter): diff --git a/tests/requirements.txt b/tests/requirements.txt index f5ebd71..be90c0a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,6 +10,6 @@ pygments moban moban_jinja2_github pyexcel -pyexcel-xls +pyexcel-xls==0.5.9 SQLAlchemy pyexcel-xlsxw diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index 9b64915..53730de 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -33,7 +33,9 @@ class TestReaders(TestCase): sheet.get_file_handle() def test_sheet_file_reader(self): - r = CSVFileReader(NamedContent(self.file_type, self.test_file)) + r = EncapsulatedSheetReader( + CSVFileReader(NamedContent(self.file_type, self.test_file)) + ) result = list(r.to_array()) self.assertEqual(result, self.expected_data) @@ -42,7 +44,9 @@ class TestReaders(TestCase): with open(self.test_file, "r") as f: io.write(f.read()) io.seek(0) - r = CSVinMemoryReader(NamedContent(self.file_type, io)) + r = EncapsulatedSheetReader( + CSVinMemoryReader(NamedContent(self.file_type, io)) + ) result = list(r.to_array()) self.assertEqual(result, self.expected_data) @@ -136,7 +140,9 @@ class TestNonUniformCSV(TestCase): def test_utf16_decoding(): test_file = os.path.join("tests", "fixtures", "csv-encoding-utf16.csv") - reader = CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") + reader = EncapsulatedSheetReader( + CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") + ) content = list(reader.to_array()) expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] @@ -160,8 +166,8 @@ def test_utf16_encoding(): def test_utf16_memory_decoding(): test_content = u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot" test_content = BytesIO(test_content.encode("utf-16")) - reader = CSVinMemoryReader( - NamedContent("csv", test_content), encoding="utf-16" + reader = EncapsulatedSheetReader( + CSVinMemoryReader(NamedContent("csv", test_content), encoding="utf-16") ) content = list(reader.to_array()) diff --git a/tests/test_django_book.py b/tests/test_django_book.py index fa231c4..5db3e7c 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -339,7 +339,7 @@ class TestMultipleModels: exporter.append(adapter1) exporter.append(adapter2) reader = DjangoBookReader(exporter, "django") - result = reader.read_all() + result = read_all(reader) for key in result: result[key] = list(result[key]) eq_(result, self.content) @@ -411,3 +411,10 @@ def test_django_model_import_adapter(): adapter.column_names = ["a"] adapter.row_initializer = "abc" eq_(adapter.row_initializer, "abc") + + +def read_all(reader): + result = OrderedDict() + for index, sheet in enumerate(reader.content_array): + result.update({sheet.name: reader.read_sheet(index).to_array()}) + return result diff --git a/tests/test_plugin_api.py b/tests/test_plugin_api.py new file mode 100644 index 0000000..94b5279 --- /dev/null +++ b/tests/test_plugin_api.py @@ -0,0 +1,53 @@ +from pyexcel_io.plugin_api import ISheet, IReader, IWriter, ISheetWriter + +from nose.tools import raises + + +class TestISheet: + def setUp(self): + self.isheet = ISheet() + + @raises(NotImplementedError) + def test_row_iterator(self): + self.isheet.row_iterator() + + @raises(NotImplementedError) + def test_column_iterator(self): + self.isheet.column_iterator(1) + + +class TestISheetWriter: + def setUp(self): + self.isheet_writer = ISheetWriter() + + @raises(NotImplementedError) + def test_write_row(self): + self.isheet_writer.write_row([1, 2]) + + +class TestIReader: + def setUp(self): + self.ireader = IReader() + + @raises(NotImplementedError) + def test_read_sheet(self): + self.ireader.read_sheet(1) + + +class TestIWriter: + def setUp(self): + self.iwriter = IWriter() + + @raises(NotImplementedError) + def test_create_sheet(self): + self.iwriter.create_sheet("a name") + + +@raises(Exception) +def test_empty_writer(): + class TestWriter(IWriter): + def create_sheet(self, sheet_name): + return None + + test_writer = TestWriter() + test_writer.write({"sheet 1": [[1, 2]]}) diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 334a781..514d813 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -230,7 +230,7 @@ class TestSingleWrite: query_sets = mysession.query(Pyexcel).all() query_reader = QueryReader(query_sets, None, column_names=self.data[0]) - result = query_reader.read_all() + result = read_all(query_reader) for key in result: result[key] = list(result[key]) eq_(result, {"pyexcel_sheet1": self.results}) @@ -471,7 +471,7 @@ class TestMultipleRead: post_adapter = SQLTableExportAdapter(Post) exporter.append(post_adapter) reader = SQLBookReader(exporter, "sql") - result = reader.read_all() + result = read_all(reader) for key in result: result[key] = list(result[key]) @@ -574,3 +574,10 @@ def test_unknown_sheet(): to_store = OrderedDict() to_store.update({"you do not see me": [[]]}) writer.write(to_store) + + +def read_all(reader): + result = OrderedDict() + for index, sheet in enumerate(reader.content_array): + result.update({sheet.name: reader.read_sheet(index).to_array()}) + return result From 9517e0bf49fbfc83d534aa8a0c0e4817a7a2ecad Mon Sep 17 00:00:00 2001 From: jaska Date: Mon, 5 Oct 2020 23:36:42 +0100 Subject: [PATCH 096/143] WIP - Document the way to write reader and writer plugins (#91) * :books: add custom plugin writer * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :books: update writer plugin * :books: update plugin writer and reader documentation * :books: telling the location of the example codes Co-authored-by: chfw --- .moban.d/docs/source/index.rst.jj2 | 1 + docs/source/conf.py | 6 +- docs/source/extensions.rst | 144 ++++++++++++++++++----- docs/source/index.rst | 1 + examples/custom_yaml_writer.py | 46 ++++++++ pyexcel-io.yml | 10 +- pyexcel_io/plugin_api/abstract_reader.py | 5 +- pyexcel_io/plugin_api/abstract_sheet.py | 7 +- setup.py | 6 +- tests/test_plugin_api.py | 8 ++ 10 files changed, 194 insertions(+), 40 deletions(-) create mode 100644 examples/custom_yaml_writer.py diff --git a/.moban.d/docs/source/index.rst.jj2 b/.moban.d/docs/source/index.rst.jj2 index bd24745..49f216e 100644 --- a/.moban.d/docs/source/index.rst.jj2 +++ b/.moban.d/docs/source/index.rst.jj2 @@ -91,6 +91,7 @@ get_data(.., library='pyexcel-ods') csvz sqlalchemy django + options extensions diff --git a/docs/source/conf.py b/docs/source/conf.py index fa602ac..b6d0920 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,16 +26,16 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'chfw' # The short X.Y version -version = '0.6.0' +version = '0.5.20' # The full version, including alpha/beta/rc tags -release = '0.5.20' +release = '0.6.0' # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode',] +extensions = [ 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', 'sphinx.ext.autodoc',] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index a9b6c68..46eb2d9 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -1,42 +1,129 @@ Extend pyexcel-io Tutorial ================================================================================ -pyexcel-io itself comes with csv support. - -Reader --------------------------------------------------------------------------------- - -Suppose we have a yaml file, containing a dictionary where the values are -two dimensional array. The task is write reader plugin to pyexcel-io so that -we can use get_data() to read it out. - -Example yaml data:: - -.. literalinclude:: ../../examples/test.yaml - :language: yaml - -Example code:: - -.. literalinclude:: ../../examples/custom_yeaml_reader.py - :language: python - - -Writer --------------------------------------------------------------------------------- - - - -Working with xls, xlsx, and ods formats -================================================================================ +You are welcome extend pyexcel-io to read and write more tabular formats. In +github repo, you will find two examples in `examples` folder. This section +explains its implementations to help you write yours. .. note:: No longer, you will need to do explicit imports for pyexcel-io extensions. Instead, you install them and manage them via pip. -Work with physical file +Reader +-------------------------------------------------------------------------------- + +Suppose we have a yaml file, containing a dictionary where the values are +two dimensional array. The task is to write a reader plugin to pyexcel-io so that +we can use get_data() to read yaml file out. + +.. literalinclude:: ../../examples/test.yaml + :language: yaml + +**Implement IReader** + +First, let's impolement reader interface as below. Three implementations are required: + +1. `content_array` attribute, is expected to be a list of `NamedContent` +2. `read_sheet` function, read sheet content by its index. +3. `close` function, to clean up any file handle + +.. literalinclude:: ../../examples/custom_yaml_reader.py + :language: python + :lines: 19-33 + +**Implement ISheet** + +`YourSingleSheet` makes this simple task complex in order to show case its inner +workings. Two abstract functions require implementation: + +1. `row_iterator`: should return a row: either content arry or content index as long as + `column_iterator` understands + +2. `column_iterator`: should return cell values one by one. + +.. literalinclude:: ../../examples/custom_yaml_reader.py + :language: python + :lines: 8-16 + + +**Plug in pyexcel-io** + +Last thing is to register with pyexcel-io about your new reader. `relative_plugin_class_path` +meant reference from current module, how to refer to `YourReader`. `locations` meant +the physical presence of the data source: "file", "memory" or "content". "file" means +files on physical disk. "memory" means a file stream. "content" means a string buffer. +`stream_type` meant the type of the stream: binary for BytesIO and text for StringIO. + +.. literalinclude:: ../../examples/custom_yaml_reader.py + :language: python + :lines: 36-41 + + +**Test your reader ** + +Let's run the following code and see if it works. + +.. literalinclude:: ../../examples/custom_yaml_reader.py + :language: python + :lines: 43-45 + +Writer +-------------------------------------------------------------------------------- + +Now for the writer, let's write a pyexcel-io writer that write a dictionary of +two dimentaional arrays back into a yaml file seen above. + +** Implement IWriter ** + +Two abstract functions are required: + +1. `create_sheet` creates a native sheet by sheet name, that understands how to code up the native sheet. Interestingly, it returns your sheet. +2. `close` function closes file handle if any. + +.. literalinclude:: ../../examples/custom_yaml_writer.py + :language: python + :lines: 18-30 + +** Implement ISheetWriter ** + +It is imagined that you will have your own sheet writer. You simply need to figure +out how to write a row. Row by row write action was already written by `ISheetWrier`. + + +.. literalinclude:: ../../examples/custom_yaml_writer.py + :language: python + :lines: 7-14 + +**Plug in pyexcel-io** + +Like the reader plugin, we register a writer. + +.. literalinclude:: ../../examples/custom_yaml_writer.py + :language: python + :lines: 33-38 + +**Test It** + +Let's run the following code and please examine `mytest.yaml` yourself. + +.. literalinclude:: ../../examples/custom_yaml_writer.py + :language: python + :lines: 40-46 + + + +Other pyexcel-io plugins ----------------------------------------------------------------------------- +Get xls support + +.. code-block:: + + + $ pip install pyexcel-xls + + Here's what is needed:: >>> from pyexcel_io import save_data @@ -71,7 +158,6 @@ And you can also get the data back:: The same applies to :meth:`pyexcel_io.get_data`. - Other formats ----------------------------------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index db51486..2d47c3c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -193,6 +193,7 @@ get_data(.., library='pyexcel-ods') csvz sqlalchemy django + options extensions diff --git a/examples/custom_yaml_writer.py b/examples/custom_yaml_writer.py new file mode 100644 index 0000000..1dc0cad --- /dev/null +++ b/examples/custom_yaml_writer.py @@ -0,0 +1,46 @@ +import yaml +from pyexcel_io import save_data +from pyexcel_io.plugins import IOPluginInfoChainV2 +from pyexcel_io.plugin_api import IWriter, ISheetWriter + + +class MySheetWriter(ISheetWriter): + def __init__(self, sheet_reference): + self.native_sheet = sheet_reference + + def write_row(self, data_row): + self.native_sheet.append(data_row) + + def close(self): + pass + + +class MyWriter(IWriter): + def __init__(self, file_name, file_type, **keywords): + self.file_name = file_name + self.content = {} + + def create_sheet(self, name): + array = [] + self.content[name] = array + return MySheetWriter(array) + + def close(self): + with open(self.file_name, "w") as f: + f.write(yaml.dump(self.content, default_flow_style=False)) + + +IOPluginInfoChainV2(__name__).add_a_writer( + relative_plugin_class_path="MyWriter", + locations=["file"], + file_types=["yaml"], + stream_type="text", +) + +if __name__ == "__main__": + data_dict = { + "sheet 1": [[1, 3, 4], [2, 4, 9]], + "sheet 2": [["B", "C", "D"]], + } + + save_data("mytest.yaml", data_dict) diff --git a/pyexcel-io.yml b/pyexcel-io.yml index ecf687a..61e1d14 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,9 +2,9 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.0 +version: 0.5.20 current_version: 0.6.0 -release: 0.5.20 +release: 0.6.0 copyright_year: 2015-2020 moban_command: false is_on_conda: true @@ -31,6 +31,12 @@ keywords: - csvz - django - sqlalchemy +sphinx_extensions: + - sphinx.ext.autosummary + - sphinx.ext.doctest + - sphinx.ext.intersphinx + - sphinx.ext.viewcode + - sphinx.ext.autodoc description: A python library to read and write structured data in csv, zipped csv format and to/from databases python_requires: ">=3.6" min_python_version: "3.6" diff --git a/pyexcel_io/plugin_api/abstract_reader.py b/pyexcel_io/plugin_api/abstract_reader.py index 12dc919..8ee4b4b 100644 --- a/pyexcel_io/plugin_api/abstract_reader.py +++ b/pyexcel_io/plugin_api/abstract_reader.py @@ -9,10 +9,13 @@ class IReader(object): """ def read_sheet(self, sheet_index) -> ISheet: - raise NotImplementedError("") + raise NotImplementedError("Read the sheet by index") def sheet_names(self): return [content.name for content in self.content_array] def __len__(self): return len(self.content_array) + + def close(self): + raise NotImplementedError("Close the file") diff --git a/pyexcel_io/plugin_api/abstract_sheet.py b/pyexcel_io/plugin_api/abstract_sheet.py index 7e82aa7..bf881e0 100644 --- a/pyexcel_io/plugin_api/abstract_sheet.py +++ b/pyexcel_io/plugin_api/abstract_sheet.py @@ -1,9 +1,9 @@ class ISheet(object): def row_iterator(self): - raise NotImplementedError("") + raise NotImplementedError("iterate each row") def column_iterator(self, row): - raise NotImplementedError("") + raise NotImplementedError("iterate each column at a given row") class ISheetWriter(object): @@ -16,3 +16,6 @@ class ISheetWriter(object): """ for row in table: self.write_row(row) + + def close(self): + raise NotImplementedError("How would you close your file") diff --git a/setup.py b/setup.py index 19ec082..88c0275 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ DESCRIPTION = ( "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.5.20.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.0.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -85,8 +85,8 @@ EXTRAS_REQUIRE = { } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs pyexcel-io v0.5.20 " + - "Find 0.5.20 in changelog for more details") +GS_COMMAND = ("gs pyexcel-io v0.6.0 " + + "Find 0.6.0 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( diff --git a/tests/test_plugin_api.py b/tests/test_plugin_api.py index 94b5279..1d591ad 100644 --- a/tests/test_plugin_api.py +++ b/tests/test_plugin_api.py @@ -24,6 +24,10 @@ class TestISheetWriter: def test_write_row(self): self.isheet_writer.write_row([1, 2]) + @raises(NotImplementedError) + def test_close(self): + self.isheet_writer.close() + class TestIReader: def setUp(self): @@ -33,6 +37,10 @@ class TestIReader: def test_read_sheet(self): self.ireader.read_sheet(1) + @raises(NotImplementedError) + def test_close(self): + self.ireader.close() + class TestIWriter: def setUp(self): From a923bce3be7ee820aef89fa0a03dad06d1789107 Mon Sep 17 00:00:00 2001 From: jaska Date: Tue, 6 Oct 2020 22:13:56 +0100 Subject: [PATCH 097/143] Bits bulbs (#92) * :books: update doc strings and plugin compactibility list * :fire: remove PY2 related code and update docs * :microscope: more test coverage --- .moban.d/docs/source/index.rst.jj2 | 3 +- docs/source/api/pyexcel_io.get_data.rst | 4 +- docs/source/api/pyexcel_io.iget_data.rst | 6 +++ docs/source/api/pyexcel_io.save_data.rst | 4 +- docs/source/common_parameters.rst | 24 ++++++--- docs/source/extensions.rst | 65 ++++++++++++++---------- docs/source/index.rst | 3 +- docs/source/options.rst | 11 ---- docs/source/renderer.rst | 1 - pyexcel_io/_compact.py | 18 ++----- pyexcel_io/io.py | 15 ++++-- tests/test_io.py | 24 +++------ tests/test_new_csvz_book.py | 3 -- tests/test_service.py | 56 ++++++++++++++++++++ 14 files changed, 146 insertions(+), 91 deletions(-) create mode 100644 docs/source/api/pyexcel_io.iget_data.rst delete mode 100644 docs/source/options.rst diff --git a/.moban.d/docs/source/index.rst.jj2 b/.moban.d/docs/source/index.rst.jj2 index 49f216e..faf8d4d 100644 --- a/.moban.d/docs/source/index.rst.jj2 +++ b/.moban.d/docs/source/index.rst.jj2 @@ -57,6 +57,7 @@ get_data(.., library='pyexcel-ods') ============= ======= ======== ======= ======== ======== ======== `pyexcel-io`_ `xls`_ `xlsx`_ `ods`_ `ods3`_ `odsr`_ `xlsxw`_ ============= ======= ======== ======= ======== ======== ======== + 0.6.0+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+ 0.5.10+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+ 0.5.1+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x @@ -91,7 +92,6 @@ get_data(.., library='pyexcel-ods') csvz sqlalchemy django - options extensions @@ -108,6 +108,7 @@ API .. autosummary:: :toctree: api/ + iget_data get_data save_data diff --git a/docs/source/api/pyexcel_io.get_data.rst b/docs/source/api/pyexcel_io.get_data.rst index 671a8d1..9f3c898 100644 --- a/docs/source/api/pyexcel_io.get_data.rst +++ b/docs/source/api/pyexcel_io.get_data.rst @@ -1,5 +1,5 @@ -pyexcel_io.get_data -=================== +pyexcel\_io.get\_data +===================== .. currentmodule:: pyexcel_io diff --git a/docs/source/api/pyexcel_io.iget_data.rst b/docs/source/api/pyexcel_io.iget_data.rst new file mode 100644 index 0000000..4b21126 --- /dev/null +++ b/docs/source/api/pyexcel_io.iget_data.rst @@ -0,0 +1,6 @@ +pyexcel\_io.iget\_data +====================== + +.. currentmodule:: pyexcel_io + +.. autofunction:: iget_data \ No newline at end of file diff --git a/docs/source/api/pyexcel_io.save_data.rst b/docs/source/api/pyexcel_io.save_data.rst index 791f311..fcece28 100644 --- a/docs/source/api/pyexcel_io.save_data.rst +++ b/docs/source/api/pyexcel_io.save_data.rst @@ -1,5 +1,5 @@ -pyexcel_io.save_data -==================== +pyexcel\_io.save\_data +====================== .. currentmodule:: pyexcel_io diff --git a/docs/source/common_parameters.rst b/docs/source/common_parameters.rst index 92926e5..3ec67af 100644 --- a/docs/source/common_parameters.rst +++ b/docs/source/common_parameters.rst @@ -2,9 +2,26 @@ Common parameters ================================================================================ +'library' option is added +-------------------------------------------------------------------------------- + +In order to have overlapping plugins co-exit, 'library' option is added to +get_data and save_data. + + +get_data only parameters +------------------------------- + +keep_trailing_empty_cells +******************************************************************************** + +default: False + +If turned on, the return data will contain trailing empty cells. + auto_dectect_datetime --------------------------------------------------------------------------------- +******************************************************************************** The datetime formats are: @@ -14,11 +31,6 @@ The datetime formats are: Any other datetime formats will be thrown as ValueError -'library' option is added --------------------------------------------------------------------------------- - -In order to have overlapping plugins co-exit, 'library' option is added to -get_data and save_data. csv only parameters -------------------------------------------------------------------------------- diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index 46eb2d9..6d55018 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -1,8 +1,12 @@ Extend pyexcel-io Tutorial ================================================================================ -You are welcome extend pyexcel-io to read and write more tabular formats. In -github repo, you will find two examples in `examples` folder. This section +You are welcome toextend pyexcel-io to read and write more tabular formats. +No. 1 rule, your plugin must have a prefix 'pyexcel_' in its module path. +For example, `pyexcel-xls` has 'pyexcel_xls' as its module path. Otherwise, +pyexcel-io will not load your plugin. + +On github, you will find two examples in `examples` folder. This section explains its implementations to help you write yours. .. note:: @@ -10,7 +14,7 @@ explains its implementations to help you write yours. No longer, you will need to do explicit imports for pyexcel-io extensions. Instead, you install them and manage them via pip. -Reader +Simple Reader for a yaml file -------------------------------------------------------------------------------- Suppose we have a yaml file, containing a dictionary where the values are @@ -60,7 +64,7 @@ files on physical disk. "memory" means a file stream. "content" means a string b :lines: 36-41 -**Test your reader ** +**Test your reader** Let's run the following code and see if it works. @@ -68,13 +72,21 @@ Let's run the following code and see if it works. :language: python :lines: 43-45 -Writer + +You would see these in standard output: + +.. code-block:: bash + + $ python custom_yaml_reader.py + OrderedDict([('sheet 1', [[1, 2, 3], [2, 3, 4]]), ('sheet 2', [['A', 'B', 'C']])]) + +A writer to write content in yaml -------------------------------------------------------------------------------- Now for the writer, let's write a pyexcel-io writer that write a dictionary of two dimentaional arrays back into a yaml file seen above. -** Implement IWriter ** +**Implement IWriter** Two abstract functions are required: @@ -85,7 +97,7 @@ Two abstract functions are required: :language: python :lines: 18-30 -** Implement ISheetWriter ** +**Implement ISheetWriter** It is imagined that you will have your own sheet writer. You simply need to figure out how to write a row. Row by row write action was already written by `ISheetWrier`. @@ -111,8 +123,25 @@ Let's run the following code and please examine `mytest.yaml` yourself. :language: python :lines: 40-46 +And you shall find a file named 'mytest.yaml': + + +.. code-block:: bash + + $ cat mytest.yaml + sheet 1: + - - 1 + - 3 + - 4 + - - 2 + - 4 + - 9 + sheet 2: + - - B + - C + - D + - Other pyexcel-io plugins ----------------------------------------------------------------------------- @@ -138,26 +167,6 @@ And you can also get the data back:: [[1, 2, 3]] -Work with memory file ------------------------------------------------------------------------------ - -Here is the sample code to work with memory file:: - - >>> from pyexcel_io.manager import get_io - >>> io = get_io("xls") - >>> data = [[1,2,3]] - >>> save_data(io, data, "xls") - -The difference is that you have mention file type if you use :meth:`pyexcel_io.save_data` - -And you can also get the data back:: - - >>> data = get_data(io, "xls") - >>> data['pyexcel_sheet1'] - [[1, 2, 3]] - -The same applies to :meth:`pyexcel_io.get_data`. - Other formats ----------------------------------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 2d47c3c..4b66b2c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -159,6 +159,7 @@ get_data(.., library='pyexcel-ods') ============= ======= ======== ======= ======== ======== ======== `pyexcel-io`_ `xls`_ `xlsx`_ `ods`_ `ods3`_ `odsr`_ `xlsxw`_ ============= ======= ======== ======= ======== ======== ======== + 0.6.0+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+ 0.5.10+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+ 0.5.1+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x @@ -193,7 +194,6 @@ get_data(.., library='pyexcel-ods') csvz sqlalchemy django - options extensions @@ -210,6 +210,7 @@ API .. autosummary:: :toctree: api/ + iget_data get_data save_data diff --git a/docs/source/options.rst b/docs/source/options.rst deleted file mode 100644 index 9ef9db4..0000000 --- a/docs/source/options.rst +++ /dev/null @@ -1,11 +0,0 @@ -Options -====================== - -Here is the documentation on the keyword options for get_data. - -keep_trailing_empty_cells ------------------------------- - -default: False - -If turned on, the return data will contain trailing empty cells. diff --git a/docs/source/renderer.rst b/docs/source/renderer.rst index 24a6f0a..2b5ab4d 100644 --- a/docs/source/renderer.rst +++ b/docs/source/renderer.rst @@ -1,5 +1,4 @@ Rendering(Formatting) the data - ================================================================================ You might want to do custom rendering on your data obtained. `row_renderer` was diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index 6b691c4..90ebd3a 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -7,16 +7,9 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ -# flake8: noqa -# pylint: disable=import-error -# pylint: disable=invalid-name -# pylint: disable=too-few-public-methods -# pylint: disable=ungrouped-imports -# pylint: disable=redefined-variable-type -import sys -import types import logging -from collections import OrderedDict +from io import BytesIO, StringIO # noqa: F401 +from collections import OrderedDict # noqa: F401 try: from logging import NullHandler @@ -27,8 +20,6 @@ except ImportError: pass -from io import BytesIO, StringIO - text_type = str irange = range @@ -48,7 +39,4 @@ def isstream(instance): def is_string(atype): """find out if a type is str or not""" - if atype == str: - return True - - return False + return atype == str diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index eba9315..1606f83 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -25,6 +25,10 @@ from pyexcel_io.exceptions import ( def iget_data(afile, file_type=None, **keywords): """Get data from an excel file source + The data has not gone into memory yet. If you use dedicated partial read + plugins, such as pyexcel-xlsxr, pyexcel-odsr, you will notice + the memory consumption drop when you work with big files. + :param afile: a file name, a file stream or actual content :param sheet_name: the name of the sheet to be loaded :param sheet_index: the index of the sheet to be loaded @@ -32,9 +36,6 @@ def iget_data(afile, file_type=None, **keywords): :param file_type: used only when filename is not a physical file name :param force_file_type: used only when filename refers to a physical file and it is intended to open it as forced file type. - :param streaming: toggles the type of returned data. The values of the - returned dictionary remain as generator if it is set - to True. Default is False. :param library: explicitly name a library for use. e.g. library='pyexcel-ods' :param auto_detect_float: defaults to True @@ -44,6 +45,7 @@ def iget_data(afile, file_type=None, **keywords): :param ignore_nan_text: various forms of 'NaN', 'nan' are ignored :param default_float_nan: choose one form of 'NaN', 'nan' :param pep_0515_off: turn off pep 0515. default to True. + :param keep_trailing_empty_cells: keep trailing columns. default to False :param keywords: any other library specific parameters :returns: an ordered dictionary """ @@ -59,7 +61,10 @@ def get_data(afile, file_type=None, streaming=None, **keywords): :param afile: a file name, a file stream or actual content :param sheet_name: the name of the sheet to be loaded :param sheet_index: the index of the sheet to be loaded + :param sheets: a list of sheet to be loaded :param file_type: used only when filename is not a physial file name + :param force_file_type: used only when filename refers to a physical file + and it is intended to open it as forced file type. :param streaming: toggles the type of returned data. The values of the returned dictionary remain as generator if it is set to True. Default is False. @@ -69,6 +74,10 @@ def get_data(afile, file_type=None, streaming=None, **keywords): :param auto_detect_int: defaults to True :param auto_detect_datetime: defaults to True :param ignore_infinity: defaults to True + :param ignore_nan_text: various forms of 'NaN', 'nan' are ignored + :param default_float_nan: choose one form of 'NaN', 'nan' + :param pep_0515_off: turn off pep 0515. default to True. + :param keep_trailing_empty_cells: keep trailing columns. default to False :param keywords: any other library specific parameters :returns: an ordered dictionary """ diff --git a/tests/test_io.py b/tests/test_io.py index e0eec0f..cf14ca4 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,5 +1,4 @@ import os -import sys import types from zipfile import BadZipfile from unittest import TestCase @@ -12,8 +11,6 @@ from pyexcel_io._compact import BytesIO, StringIO, OrderedDict, is_string from nose.tools import eq_, raises -PY2 = sys.version_info[0] == 2 - @raises(IOError) def test_directory_name_as_file(): @@ -116,11 +113,8 @@ def test_load_unknown_data_from_memory(): @raises(BadZipfile) def test_load_csvz_data_from_memory(): - if not PY2: - io = StringIO() - get_data(io, file_type="csvz") - else: - raise BadZipfile("pass it") + io = StringIO() + get_data(io, file_type="csvz") @raises(IOError) @@ -130,12 +124,9 @@ def test_write_xlsx_data(): @raises(Exception) def test_writer_csvz_data_from_memory(): - if not PY2: - io = StringIO() - writer = get_writer(io, file_type="csvz") - writer.write({"adb": [[2, 3]]}) - else: - raise Exception("pass it") + io = StringIO() + writer = get_writer(io, file_type="csvz") + writer.write({"adb": [[2, 3]]}) @raises(exceptions.NoSupportingPluginFound) @@ -264,10 +255,7 @@ def test_conversion_from_bytes_to_text(): def test_is_string(): - if PY2: - assert is_string(type(u"a")) is True - else: - assert is_string(type("a")) is True + assert is_string(type("a")) is True def test_generator_is_obtained(): diff --git a/tests/test_new_csvz_book.py b/tests/test_new_csvz_book.py index 6f42b13..2d4c4ef 100644 --- a/tests/test_new_csvz_book.py +++ b/tests/test_new_csvz_book.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import os -import sys import zipfile from unittest import TestCase @@ -12,8 +11,6 @@ from pyexcel_io._compact import OrderedDict from nose.tools import raises -PY2 = sys.version_info[0] == 2 - class TestCSVZ(TestCase): file_type = "csvz" diff --git a/tests/test_service.py b/tests/test_service.py index 8d0af9a..7ded3d6 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,10 +1,17 @@ +from datetime import time, datetime, timedelta + from pyexcel_io.service import ( date_value, time_value, + boolean_value, + ods_bool_value, + ods_date_value, + ods_time_value, ods_float_value, throw_exception, detect_int_value, detect_float_value, + ods_timedelta_value, ) from pyexcel_io.exceptions import IntegerAccuracyLossError @@ -106,3 +113,52 @@ def test_big_int_value(): @raises(IntegerAccuracyLossError) def test_throw_exception(): throw_exception(1000000000000000) + + +def test_boolean_value(): + fixture = ["true", "false", 1] + expected = [True, False, 1] + + actual = [boolean_value(element) for element in fixture] + eq_(actual, expected) + + +def test_time_delta_presentation(): + a = datetime(2020, 12, 12, 12, 12, 12) + b = datetime(2020, 11, 12, 12, 12, 11) + delta = a - b + + value = ods_timedelta_value(delta) + eq_(value, "PT720H00M01S") + + +def test_ods_bool_to_string(): + fixture = [True, False] + expected = ["true", "false"] + + actual = [ods_bool_value(element) for element in fixture] + eq_(actual, expected) + + +def test_ods_time_value(): + test = datetime(2020, 10, 6, 11, 11, 11) + actual = ods_time_value(test) + eq_(actual, "PT11H11M11S") + + +def test_ods_date_value(): + test = datetime(2020, 10, 6, 11, 11, 11) + actual = ods_date_value(test) + eq_(actual, "2020-10-06") + + +def test_time_value_returns_time_delta(): + test_time_value = "PT720H00M01S" + delta = time_value(test_time_value) + eq_(delta, timedelta(days=30, seconds=1)) + + +def test_time_value(): + test_time_value = "PT23H00M01S" + delta = time_value(test_time_value) + eq_(delta, time(23, 0, 1)) From c684c08efa610da8e93d4b19e540eff5ddd04dfe Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 6 Oct 2020 23:12:24 +0100 Subject: [PATCH 098/143] :hammer: shift NamedContent to plugin interface --- docs/source/extensions.rst | 2 +- pyexcel_io/plugin_api/__init__.py | 2 +- pyexcel_io/plugin_api/abstract_sheet.py | 10 ++++++++++ pyexcel_io/sheet.py | 11 +---------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index 6d55018..d3d0ed3 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -1,7 +1,7 @@ Extend pyexcel-io Tutorial ================================================================================ -You are welcome toextend pyexcel-io to read and write more tabular formats. +You are welcome to extend pyexcel-io to read and write more tabular formats. No. 1 rule, your plugin must have a prefix 'pyexcel_' in its module path. For example, `pyexcel-xls` has 'pyexcel_xls' as its module path. Otherwise, pyexcel-io will not load your plugin. diff --git a/pyexcel_io/plugin_api/__init__.py b/pyexcel_io/plugin_api/__init__.py index bb39d55..4d2d771 100644 --- a/pyexcel_io/plugin_api/__init__.py +++ b/pyexcel_io/plugin_api/__init__.py @@ -1,3 +1,3 @@ -from .abstract_sheet import ISheet, ISheetWriter # noqa: F401 +from .abstract_sheet import ISheet, ISheetWriter, NamedContent # noqa: F401 from .abstract_reader import IReader # noqa: F401 from .abstract_writer import IWriter # noqa: F401 diff --git a/pyexcel_io/plugin_api/abstract_sheet.py b/pyexcel_io/plugin_api/abstract_sheet.py index bf881e0..ff81b76 100644 --- a/pyexcel_io/plugin_api/abstract_sheet.py +++ b/pyexcel_io/plugin_api/abstract_sheet.py @@ -19,3 +19,13 @@ class ISheetWriter(object): def close(self): raise NotImplementedError("How would you close your file") + + +class NamedContent(object): + """ + Helper class for content that does not have a name + """ + + def __init__(self, name, payload): + self.name = name + self.payload = payload diff --git a/pyexcel_io/sheet.py b/pyexcel_io/sheet.py index b23f342..b7c7015 100644 --- a/pyexcel_io/sheet.py +++ b/pyexcel_io/sheet.py @@ -10,16 +10,7 @@ import pyexcel_io.constants as constants from pyexcel_io.utils import _index_filter from pyexcel_io._compact import irange - - -class NamedContent(object): - """ - Helper class for content that does not have a name - """ - - def __init__(self, name, payload): - self.name = name - self.payload = payload +from pyexcel_io.plugin_api import NamedContent # noqa: F401 class SheetReader(object): From b3cb1a05c8fdf17863127398bfdc799b68a13ccf Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Oct 2020 19:49:53 +0100 Subject: [PATCH 099/143] :hammer: update code for denepdency injection and add user warning about older interface --- pyexcel_io/book.py | 9 +++++++++ pyexcel_io/reader.py | 34 ++++++++++++++++++++++------------ pyexcel_io/writer.py | 35 +++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 24 deletions(-) diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index d234329..7f0baa1 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -7,11 +7,19 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +import warnings + import pyexcel_io.manager as manager from pyexcel_io._compact import OrderedDict, isstream from .constants import MESSAGE_ERROR_03, MESSAGE_WRONG_IO_INSTANCE +DEPRECATED_SINCE_0_6_0 = ( + "Deprecated since v0.6.0! " + + "Although backward compatibility is preserved, " + + "it is recommended to upgrade to get new features." +) + class RWInterface(object): """ @@ -21,6 +29,7 @@ class RWInterface(object): stream_type = None def __init__(self): + warnings.warn(DEPRECATED_SINCE_0_6_0) self._file_type = None def open(self, file_name, **keywords): diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py index 704a709..ee45c89 100644 --- a/pyexcel_io/reader.py +++ b/pyexcel_io/reader.py @@ -33,12 +33,20 @@ class Reader(object): self.library = library self.keywords = None + # if you know which reader class to use, this attribute allows + # you to set reader class externally. Since there is no + # so call private field in Python, I am not going to create + # useless setter and getter functions like Java. + # in pyexcel, this attribute is mainly used for testing + self.reader_class = None + def open(self, file_name, **keywords): - reader_class = NEW_READERS.get_a_plugin( - self.file_type, location="file", library=self.library - ) + if self.reader_class is None: + self.reader_class = NEW_READERS.get_a_plugin( + self.file_type, location="file", library=self.library + ) self.keywords, native_sheet_keywords = clean_keywords(keywords) - self.reader = reader_class( + self.reader = self.reader_class( file_name, self.file_type, **native_sheet_keywords ) return self.reader @@ -46,10 +54,11 @@ class Reader(object): def open_content(self, file_content, **keywords): self.keywords, native_sheet_keywords = clean_keywords(keywords) try: - reader_class = NEW_READERS.get_a_plugin( - self.file_type, location="content", library=self.library - ) - self.reader = reader_class( + if self.reader_class is None: + self.reader_class = NEW_READERS.get_a_plugin( + self.file_type, location="content", library=self.library + ) + self.reader = self.reader_class( file_content, self.file_type, **native_sheet_keywords ) return self.reader @@ -64,10 +73,11 @@ class Reader(object): def open_stream(self, file_stream, **keywords): self.keywords, native_sheet_keywords = clean_keywords(keywords) - reader_class = NEW_READERS.get_a_plugin( - self.file_type, location="memory", library=self.library - ) - self.reader = reader_class( + if self.reader_class is None: + self.reader_class = NEW_READERS.get_a_plugin( + self.file_type, location="memory", library=self.library + ) + self.reader = self.reader_class( file_stream, self.file_type, **native_sheet_keywords ) return self.reader diff --git a/pyexcel_io/writer.py b/pyexcel_io/writer.py index 700d86e..5c41b7f 100644 --- a/pyexcel_io/writer.py +++ b/pyexcel_io/writer.py @@ -6,26 +6,37 @@ class Writer(object): self.file_type = file_type self.library = library self.keyboards = None + # if you know which reader class to use, this attribute allows + # you to set reader class externally. Since there is no + # so call private field in Python, I am not going to create + # useless setter and getter functions like Java. + # in pyexcel, this attribute is mainly used for testing + self.writer_class = None def open(self, file_name, **keywords): - writer_class = NEW_WRITERS.get_a_plugin( - self.file_type, library=self.library, location="file" - ) - self.writer = writer_class(file_name, self.file_type, **keywords) + if self.writer_class is None: + self.writer_class = NEW_WRITERS.get_a_plugin( + self.file_type, library=self.library, location="file" + ) + self.writer = self.writer_class(file_name, self.file_type, **keywords) def open_content(self, file_stream, **keywords): - # if not isstream(file_stream): - # raise IOError(MESSAGE_ERROR_03) - writer_class = NEW_WRITERS.get_a_plugin( - self.file_type, library=self.library, location="content" + if self.writer_class is None: + self.writer_class = NEW_WRITERS.get_a_plugin( + self.file_type, library=self.library, location="content" + ) + self.writer = self.writer_class( + file_stream, self.file_type, **keywords ) - self.writer = writer_class(file_stream, self.file_type, **keywords) def open_stream(self, file_stream, **keywords): - writer_class = NEW_WRITERS.get_a_plugin( - self.file_type, library=self.library, location="memory" + if self.writer_class is None: + self.writer_class = NEW_WRITERS.get_a_plugin( + self.file_type, library=self.library, location="memory" + ) + self.writer = self.writer_class( + file_stream, self.file_type, **keywords ) - self.writer = writer_class(file_stream, self.file_type, **keywords) def write(self, incoming_dict): self.writer.write(incoming_dict) From 233045f889454889de32a75a08581dfa8b7678b4 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Oct 2020 22:58:17 +0100 Subject: [PATCH 100/143] :bug: fix the potential to leave unclosed handle. --- pyexcel_io/readers/csv_in_file.py | 1 + pyexcel_io/writers/csv_in_file.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pyexcel_io/readers/csv_in_file.py b/pyexcel_io/readers/csv_in_file.py index 8e76930..b926242 100644 --- a/pyexcel_io/readers/csv_in_file.py +++ b/pyexcel_io/readers/csv_in_file.py @@ -61,3 +61,4 @@ class FileReader(IReader): def close(self): for reader in self.handles: reader.close() + self.handles = [] diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py index 21eb85c..46b23ac 100644 --- a/pyexcel_io/writers/csv_in_file.py +++ b/pyexcel_io/writers/csv_in_file.py @@ -10,18 +10,20 @@ class CsvFileWriter(IWriter): if file_type == constants.FILE_FORMAT_TSV: self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT self.__index = 0 - self.writer = None + self.handlers = [] def create_sheet(self, name): - self.writer = CSVFileWriter( + writer = CSVFileWriter( self._file_alike_object, name, sheet_index=self.__index, **self._keywords ) self.__index = self.__index + 1 - return self.writer + self.handlers.append(writer) + return writer def close(self): - if self.writer: - self.writer.close() + for writer in self.handlers: + writer.close() + self.handlers = [] From bc4d2b33407238f7782d24aa018e1e81d605af3d Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Oct 2020 23:01:07 +0100 Subject: [PATCH 101/143] :egg: :ferris_wheel: release 0.6.0 --- CHANGELOG.rst | 22 +++++++++++----------- changelog.yml | 14 +++++++------- docs/source/conf.py | 2 +- docs/source/index.rst | 1 - pyexcel-io.yml | 3 +-- requirements.txt | 1 - 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53e473e..cd42919 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,19 +1,9 @@ Change log ================================================================================ -0.6.0 - tbd +0.6.0 - 7.10.2020 -------------------------------------------------------------------------------- -**added** - -#. `#86 `_: allow trailing - options, get_data(...keep_trailing_empty_cells=True). - -**fixed** - -#. `#74 `_: handle zip files - which contain non-UTF-8 encoded files. - **removed** #. python 3.6 lower versions are no longer supported @@ -25,6 +15,16 @@ Change log #. new query set reader plugin. pyexcel<=0.6.4 has used intrusive way of getting query set source done. it is against the plugin interface. +**fixed** + +#. `#74 `_: handle zip files + which contain non-UTF-8 encoded files. + +**added** + +#. `#86 `_: allow trailing + options, get_data(...keep_trailing_empty_cells=True). + 0.5.20 - 17.7.2019 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index ea6d4bd..f924250 100644 --- a/changelog.yml +++ b/changelog.yml @@ -2,12 +2,6 @@ name: pyexcel-io organisation: pyexcel releases: - changes: - - action: added - details: - - "`#86`: allow trailing options, get_data(...keep_trailing_empty_cells=True)." - - action: fixed - details: - - "`#74`: handle zip files which contain non-UTF-8 encoded files." - action: removed details: - 'python 3.6 lower versions are no longer supported' @@ -17,8 +11,14 @@ releases: please read 'Packaging with Pyinstaller' in the documentation. - new query set reader plugin. pyexcel<=0.6.4 has used intrusive way of getting query set source done. it is against the plugin interface. + - action: fixed + details: + - "`#74`: handle zip files which contain non-UTF-8 encoded files." + - action: added + details: + - "`#86`: allow trailing options, get_data(...keep_trailing_empty_cells=True)." version: 0.6.0 - date: tbd + date: 7.10.2020 - changes: - action: updated details: diff --git a/docs/source/conf.py b/docs/source/conf.py index b6d0920..9588316 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'chfw' # The short X.Y version -version = '0.5.20' +version = '0.6.0' # The full version, including alpha/beta/rc tags release = '0.6.0' diff --git a/docs/source/index.rst b/docs/source/index.rst index 4b66b2c..95be968 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,6 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License -:Development: |release| :Released: |version| :Generated: |today| diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 61e1d14..fea0582 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,14 +2,13 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.5.20 +version: 0.6.0 current_version: 0.6.0 release: 0.6.0 copyright_year: 2015-2020 moban_command: false is_on_conda: true dependencies: - - ordereddict;python_version<"2.7" - lml>=0.0.4 test_dependencies: - pyexcel diff --git a/requirements.txt b/requirements.txt index dac001a..8539e81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -ordereddict;python_version<"2.7" lml>=0.0.4 From 2ebe9559add925cac80ac4e3951c648316562de0 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Oct 2020 23:31:51 +0100 Subject: [PATCH 102/143] :egg: :ferris_wheel: use 0.6.1 for 0.6.0 because 0.6.0 was occupied --- CHANGELOG.rst | 2 +- changelog.yml | 2 +- docs/source/conf.py | 4 ++-- pyexcel-io.yml | 6 +++--- setup.py | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cd42919..50a44c7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Change log ================================================================================ -0.6.0 - 7.10.2020 +0.6.1 - 7.10.2020 -------------------------------------------------------------------------------- **removed** diff --git a/changelog.yml b/changelog.yml index f924250..0e0039b 100644 --- a/changelog.yml +++ b/changelog.yml @@ -17,7 +17,7 @@ releases: - action: added details: - "`#86`: allow trailing options, get_data(...keep_trailing_empty_cells=True)." - version: 0.6.0 + version: 0.6.1 date: 7.10.2020 - changes: - action: updated diff --git a/docs/source/conf.py b/docs/source/conf.py index 9588316..a30ee98 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,9 +26,9 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'chfw' # The short X.Y version -version = '0.6.0' +version = '0.6.1' # The full version, including alpha/beta/rc tags -release = '0.6.0' +release = '0.6.1' # -- General configuration --------------------------------------------------- diff --git a/pyexcel-io.yml b/pyexcel-io.yml index fea0582..8c837d9 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,9 +2,9 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.0 -current_version: 0.6.0 -release: 0.6.0 +version: 0.6.1 +current_version: 0.6.1 +release: 0.6.1 copyright_year: 2015-2020 moban_command: false is_on_conda: true diff --git a/setup.py b/setup.py index 88c0275..0b9a547 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "chfw" -VERSION = "0.6.0" +VERSION = "0.6.1" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( @@ -40,7 +40,7 @@ DESCRIPTION = ( "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.6.0.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.1.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -85,8 +85,8 @@ EXTRAS_REQUIRE = { } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs pyexcel-io v0.6.0 " + - "Find 0.6.0 in changelog for more details") +GS_COMMAND = ("gs pyexcel-io v0.6.1 " + + "Find 0.6.1 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( From 5afbe747f459d0b04a45e88b4552f6397ba0dddf Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 8 Oct 2020 00:22:38 +0100 Subject: [PATCH 103/143] :bug: try support pyexcel-xls 0.4.1 on python 3.6. fix #94 --- changelog.yml | 6 ++++++ docs/source/common_parameters.rst | 4 ++-- docs/source/conf.py | 4 ++-- pyexcel-io.yml | 6 +++--- pyexcel_io/_compact.py | 2 ++ setup.py | 8 ++++---- 6 files changed, 19 insertions(+), 11 deletions(-) diff --git a/changelog.yml b/changelog.yml index 0e0039b..cdff924 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - "`#94`: keep backward compatibility for pyexcel-xls 0.4.1" + version: 0.6.2 + date: 7.10.2020 - changes: - action: removed details: diff --git a/docs/source/common_parameters.rst b/docs/source/common_parameters.rst index 3ec67af..70e01c9 100644 --- a/docs/source/common_parameters.rst +++ b/docs/source/common_parameters.rst @@ -5,12 +5,12 @@ Common parameters 'library' option is added -------------------------------------------------------------------------------- -In order to have overlapping plugins co-exit, 'library' option is added to +In order to have overlapping plugins co-exist, 'library' option is added to get_data and save_data. get_data only parameters -------------------------------- +-------------------------------------------------------------------------------- keep_trailing_empty_cells ******************************************************************************** diff --git a/docs/source/conf.py b/docs/source/conf.py index a30ee98..2828238 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,9 +26,9 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'chfw' # The short X.Y version -version = '0.6.1' +version = '0.6.2' # The full version, including alpha/beta/rc tags -release = '0.6.1' +release = '0.6.2' # -- General configuration --------------------------------------------------- diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 8c837d9..8e44348 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,9 +2,9 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.1 -current_version: 0.6.1 -release: 0.6.1 +version: 0.6.2 +current_version: 0.6.2 +release: 0.6.2 copyright_year: 2015-2020 moban_command: false is_on_conda: true diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index 90ebd3a..1e0fa18 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -7,6 +7,7 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +import sys import logging from io import BytesIO, StringIO # noqa: F401 from collections import OrderedDict # noqa: F401 @@ -22,6 +23,7 @@ except ImportError: text_type = str irange = range +PY2 = sys.version[0] == 2 def isstream(instance): diff --git a/setup.py b/setup.py index 0b9a547..c03f51a 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "chfw" -VERSION = "0.6.1" +VERSION = "0.6.2" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( @@ -40,7 +40,7 @@ DESCRIPTION = ( "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.6.1.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.2.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -85,8 +85,8 @@ EXTRAS_REQUIRE = { } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs pyexcel-io v0.6.1 " + - "Find 0.6.1 in changelog for more details") +GS_COMMAND = ("gs pyexcel-io v0.6.2 " + + "Find 0.6.2 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( From b46dc168837b7917515c9e757854ef49e0a927b5 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 7 Oct 2020 23:23:16 +0000 Subject: [PATCH 104/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 50a44c7..6b8f1e7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.6.2 - 7.10.2020 +-------------------------------------------------------------------------------- + +**updated** + +#. `#94 `_: keep backward + compatibility for pyexcel-xls 0.4.1 + 0.6.1 - 7.10.2020 -------------------------------------------------------------------------------- From b942935169bb3a1f1028e716e409f2158cc07623 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 10 Oct 2020 14:39:53 +0100 Subject: [PATCH 105/143] :lipstick: kick start 0.6.3 --- CHANGELOG.rst | 8 ++++++++ LICENSE | 2 +- README.rst | 5 +++++ docs/source/conf.py | 2 +- docs/source/index.rst | 1 + pyexcel-io.yml | 4 ++-- setup.py | 8 ++++---- 7 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 50a44c7..6b8f1e7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.6.2 - 7.10.2020 +-------------------------------------------------------------------------------- + +**updated** + +#. `#94 `_: keep backward + compatibility for pyexcel-xls 0.4.1 + 0.6.1 - 7.10.2020 -------------------------------------------------------------------------------- diff --git a/LICENSE b/LICENSE index 802c39c..747c8b4 100644 --- a/LICENSE +++ b/LICENSE @@ -13,7 +13,7 @@ that the following conditions are met: and/or other materials provided with the distribution. * Neither the name of 'pyexcel-io' nor the names of the contributors - may be used to endorse or promote products derived from this software + may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND diff --git a/README.rst b/README.rst index da577f8..2f9b838 100644 --- a/README.rst +++ b/README.rst @@ -29,6 +29,11 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg :target: https://gitter.im/pyexcel/Lobby +.. image:: https://img.shields.io/static/v1?label=continuous%20templating&message=%E6%A8%A1%E7%89%88%E6%9B%B4%E6%96%B0&color=blue&style=flat-square + :target: https://moban.readthedocs.io/en/latest/#at-scale-continous-templating-for-open-source-projects + +.. image:: https://img.shields.io/static/v1?label=coding%20style&message=black&color=black&style=flat-square + :target: https://github.com/psf/black .. image:: https://readthedocs.org/projects/pyexcel-io/badge/?version=latest :target: http://pyexcel-io.readthedocs.org/en/latest/ diff --git a/docs/source/conf.py b/docs/source/conf.py index 2828238..9c84362 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ author = 'chfw' # The short X.Y version version = '0.6.2' # The full version, including alpha/beta/rc tags -release = '0.6.2' +release = '0.6.3' # -- General configuration --------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 95be968..4b66b2c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License +:Development: |release| :Released: |version| :Generated: |today| diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 8e44348..f344f13 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -3,8 +3,8 @@ project: "pyexcel-io" name: pyexcel-io nick_name: io version: 0.6.2 -current_version: 0.6.2 -release: 0.6.2 +current_version: 0.6.3 +release: 0.6.3 copyright_year: 2015-2020 moban_command: false is_on_conda: true diff --git a/setup.py b/setup.py index c03f51a..6c9261b 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "chfw" -VERSION = "0.6.2" +VERSION = "0.6.3" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( @@ -40,7 +40,7 @@ DESCRIPTION = ( "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.6.2.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.3.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -85,8 +85,8 @@ EXTRAS_REQUIRE = { } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs pyexcel-io v0.6.2 " + - "Find 0.6.2 in changelog for more details") +GS_COMMAND = ("gs pyexcel-io v0.6.3 " + + "Find 0.6.3 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( From 71c28b4bc911597fd8cfaa7d5182bf4d70e885dd Mon Sep 17 00:00:00 2001 From: jaska Date: Mon, 12 Oct 2020 22:56:58 +0100 Subject: [PATCH 106/143] Issue 96 (#97) * :bug: regression: unknown file type shall trigger NoSupportingPluginFound. fix #96 * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :green_heart: update coding style. #96 * :microscope: more test coverage Co-authored-by: chfw --- CHANGELOG.rst | 8 ++++++++ changelog.yml | 6 ++++++ pyexcel_io/exceptions.py | 6 ------ pyexcel_io/reader.py | 25 +++++++------------------ pyexcel_io/readers/__init__.py | 5 +++++ pyexcel_io/readers/csvz.py | 7 +++++++ tests/test_issues.py | 22 +++++++--------------- tests/test_service.py | 6 ++++++ 8 files changed, 46 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6b8f1e7..95587f5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.6.3 - tbd +-------------------------------------------------------------------------------- + +**updated** + +#. `#96 `_: regression: unknown + file type shall trigger NoSupportingPluginFound + 0.6.2 - 7.10.2020 -------------------------------------------------------------------------------- diff --git a/changelog.yml b/changelog.yml index cdff924..15e5619 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - "`#96`: regression: unknown file type shall trigger NoSupportingPluginFound" + version: 0.6.3 + date: tbd - changes: - action: updated details: diff --git a/pyexcel_io/exceptions.py b/pyexcel_io/exceptions.py index 8a8d1f6..b6d00aa 100644 --- a/pyexcel_io/exceptions.py +++ b/pyexcel_io/exceptions.py @@ -21,12 +21,6 @@ class SupportingPluginAvailableButNotInstalled(Exception): pass -class UpgradePlugin(Exception): - """raised when a known plugin is not compatible""" - - pass - - class IntegerAccuracyLossError(Exception): """ When an interger is greater than 999999999999999, ods loses its accuracy. diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py index ee45c89..f0276fa 100644 --- a/pyexcel_io/reader.py +++ b/pyexcel_io/reader.py @@ -1,5 +1,3 @@ -from pyexcel_io import exceptions -from pyexcel_io.book import _convert_content_to_stream from pyexcel_io.sheet import SheetReader from pyexcel_io.plugins import NEW_READERS from pyexcel_io._compact import OrderedDict @@ -53,23 +51,14 @@ class Reader(object): def open_content(self, file_content, **keywords): self.keywords, native_sheet_keywords = clean_keywords(keywords) - try: - if self.reader_class is None: - self.reader_class = NEW_READERS.get_a_plugin( - self.file_type, location="content", library=self.library - ) - self.reader = self.reader_class( - file_content, self.file_type, **native_sheet_keywords + if self.reader_class is None: + self.reader_class = NEW_READERS.get_a_plugin( + self.file_type, location="content", library=self.library ) - return self.reader - except ( - exceptions.NoSupportingPluginFound, - exceptions.SupportingPluginAvailableButNotInstalled, - ): - file_stream = _convert_content_to_stream( - file_content, self.file_type - ) - return self.open_stream(file_stream, **native_sheet_keywords) + self.reader = self.reader_class( + file_content, self.file_type, **native_sheet_keywords + ) + return self.reader def open_stream(self, file_stream, **keywords): self.keywords, native_sheet_keywords = clean_keywords(keywords) diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index 063674c..e06ec19 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -29,4 +29,9 @@ IOPluginInfoChainV2(__name__).add_a_reader( file_types=["csvz", "tsvz"], locations=["file", "memory"], stream_type="binary", +).add_a_reader( + relative_plugin_class_path="csvz.ContentReader", + file_types=["csvz", "tsvz"], + locations=["content"], + stream_type="binary", ) diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index 9fdc55e..4556015 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ import zipfile +from io import BytesIO import chardet from pyexcel_io import constants @@ -48,6 +49,12 @@ class FileReader(IReader): return CSVinMemoryReader(NamedContent(name, sheet), **self.keywords) +class ContentReader(FileReader): + def __init__(self, file_content, file_type, **keywords): + io = BytesIO(file_content) + super().__init__(io, file_type, **keywords) + + def _get_sheet_name(filename): len_of_a_dot = 1 len_of_csv_word = 3 diff --git a/tests/test_issues.py b/tests/test_issues.py index b7b9dd5..50dc537 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -1,13 +1,11 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - import os import pyexcel as p from pyexcel_io import get_data, save_data +from pyexcel_io.exceptions import NoSupportingPluginFound from nose import SkipTest -from nose.tools import eq_ +from nose.tools import eq_, raises IN_TRAVIS = "TRAVIS" in os.environ @@ -41,17 +39,6 @@ def test_issue_23(): eq_(data["issue23.csv"], expected) -# def test_issue_28(): -# from pyexcel_io.plugins import readers -# from pyexcel_io.exceptions import UpgradePlugin -# expected = "Please upgrade the plugin '%s' according to " -# expected += "plugin compactibility table." -# try: -# readers.load_me_later('pyexcel_test') -# except UpgradePlugin as e: -# eq_(str(e), expected % 'pyexcel_test') - - def test_issue_33_34(): import mmap @@ -154,5 +141,10 @@ def test_pyexcel_issue_138(): os.unlink("test.csv") +@raises(NoSupportingPluginFound) +def test_issue_96(): + get_data("foo-bar-data", file_type="Idonotexist") + + def get_fixture(file_name): return os.path.join("tests", "fixtures", file_name) diff --git a/tests/test_service.py b/tests/test_service.py index 7ded3d6..70f6209 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -3,6 +3,7 @@ from datetime import time, datetime, timedelta from pyexcel_io.service import ( date_value, time_value, + float_value, boolean_value, ods_bool_value, ods_date_value, @@ -162,3 +163,8 @@ def test_time_value(): test_time_value = "PT23H00M01S" delta = time_value(test_time_value) eq_(delta, time(23, 0, 1)) + + +def test_float_value(): + a = float_value("1.2") + eq_(a, 1.2) From 6b9b0f7e3f78ecc5236b2ee779d2b176646c71dd Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 12 Oct 2020 22:58:12 +0100 Subject: [PATCH 107/143] :egg: :ferris_wheel: release 0.6.3 --- changelog.yml | 4 ++-- tests/test_issues.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.yml b/changelog.yml index 15e5619..e53cad9 100644 --- a/changelog.yml +++ b/changelog.yml @@ -2,11 +2,11 @@ name: pyexcel-io organisation: pyexcel releases: - changes: - - action: updated + - action: fixed details: - "`#96`: regression: unknown file type shall trigger NoSupportingPluginFound" version: 0.6.3 - date: tbd + date: 12.10.2020 - changes: - action: updated details: diff --git a/tests/test_issues.py b/tests/test_issues.py index 50dc537..d9cbfd6 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -51,7 +51,7 @@ def test_issue_33_34(): def test_issue_30_utf8_BOM_header(): - content = [[u"人有悲歡離合", u"月有陰晴圓缺"]] + content = [["人有悲歡離合", "月有陰晴圓缺"]] test_file = "test-utf8-BOM.csv" save_data(test_file, content, encoding="utf-8-sig", lineterminator="\n") custom_encoded_content = get_data(test_file, encoding="utf-8-sig") From 12e1750e36f80bfd70a233b861f948a83b2fe09f Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 12 Oct 2020 21:58:41 +0000 Subject: [PATCH 108/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 95587f5..f3226d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,10 +1,10 @@ Change log ================================================================================ -0.6.3 - tbd +0.6.3 - 12.10.2020 -------------------------------------------------------------------------------- -**updated** +**fixed** #. `#96 `_: regression: unknown file type shall trigger NoSupportingPluginFound From 23d278c5f145d39103c63718780285800f6d39a7 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 12 Oct 2020 23:04:32 +0100 Subject: [PATCH 109/143] :books: update release version --- pyexcel-io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexcel-io.yml b/pyexcel-io.yml index f344f13..4dc5202 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,7 +2,7 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.2 +version: 0.6.3 current_version: 0.6.3 release: 0.6.3 copyright_year: 2015-2020 From db2faac409e5fbedcb8622340f0109cd3360782e Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 12 Oct 2020 23:06:18 +0100 Subject: [PATCH 110/143] :books: update extra dependencies --- changelog.yml | 3 +++ pyexcel-io.yml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/changelog.yml b/changelog.yml index e53cad9..47f4053 100644 --- a/changelog.yml +++ b/changelog.yml @@ -5,6 +5,9 @@ releases: - action: fixed details: - "`#96`: regression: unknown file type shall trigger NoSupportingPluginFound" + - action: updated + details: + - "extra dependencies uses 0.6.0 based plugins" version: 0.6.3 date: 12.10.2020 - changes: diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 4dc5202..8b43aff 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -17,11 +17,11 @@ test_dependencies: - pyexcel-xlsxw extra_dependencies: - xls: - - pyexcel-xls>=0.5.0 + - pyexcel-xls>=0.6.0 - xlsx: - - pyexcel-xlsx>=0.5.0 + - pyexcel-xlsx>=0.6.0 - ods: - - pyexcel-ods3>=0.5.0 + - pyexcel-ods3>=0.6.0 keywords: - API - tsv From 7adcec9bd008590c36d8b970b8e4c1d41daa6afa Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 12 Oct 2020 22:06:53 +0000 Subject: [PATCH 111/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- CHANGELOG.rst | 4 ++++ docs/source/conf.py | 2 +- docs/source/index.rst | 1 - setup.py | 6 +++--- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f3226d0..92e656b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,10 @@ Change log #. `#96 `_: regression: unknown file type shall trigger NoSupportingPluginFound +**updated** + +#. extra dependencies uses 0.6.0 based plugins + 0.6.2 - 7.10.2020 -------------------------------------------------------------------------------- diff --git a/docs/source/conf.py b/docs/source/conf.py index 9c84362..a2fc48f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'chfw' # The short X.Y version -version = '0.6.2' +version = '0.6.3' # The full version, including alpha/beta/rc tags release = '0.6.3' diff --git a/docs/source/index.rst b/docs/source/index.rst index 4b66b2c..95be968 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,6 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License -:Development: |release| :Released: |version| :Generated: |today| diff --git a/setup.py b/setup.py index 6c9261b..b5e4bea 100644 --- a/setup.py +++ b/setup.py @@ -79,9 +79,9 @@ SETUP_COMMANDS = {} PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests", "tests.*"]) EXTRAS_REQUIRE = { - "xls": ['pyexcel-xls>=0.5.0'], - "xlsx": ['pyexcel-xlsx>=0.5.0'], - "ods": ['pyexcel-ods3>=0.5.0'], + "xls": ['pyexcel-xls>=0.6.0'], + "xlsx": ['pyexcel-xlsx>=0.6.0'], + "ods": ['pyexcel-ods3>=0.6.0'], } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) From 04d28a5b2e1b7f147964b6742fb7f40568748093 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 16 Oct 2020 22:39:09 +0100 Subject: [PATCH 112/143] :handshake: update project meta data --- CHANGELOG.rst | 4 +++ README.rst | 61 +++++++++++++++++++++++------------------- docs/source/conf.py | 2 +- docs/source/index.rst | 62 +++++++++++++++++++++++-------------------- lint.sh | 2 +- setup.py | 10 +++---- 6 files changed, 77 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f3226d0..92e656b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,10 @@ Change log #. `#96 `_: regression: unknown file type shall trigger NoSupportingPluginFound +**updated** + +#. extra dependencies uses 0.6.0 based plugins + 0.6.2 - 7.10.2020 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 2f9b838..9ea17e3 100644 --- a/README.rst +++ b/README.rst @@ -74,44 +74,47 @@ sqlalchemy supported databases. Its supported file formats are extended to cover .. table:: A list of file formats supported by external plugins - ======================== ======================= ================= ================== - Package name Supported file formats Dependencies Python versions - ======================== ======================= ================= ================== - `pyexcel-io`_ >=v0.6.0 csv, csvz [#f1]_, tsv, 3.6+ - tsvz [#f2]_ - `pyexcel-io`_ <=0.5.20 same as above 2.6, 2.7, 3.3, - 3.4, 3.5, 3.6 - pypy - `pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, same as above + ======================== ======================= ================= + Package name Supported file formats Dependencies + ======================== ======================= ================= + `pyexcel-io`_ csv, csvz [#f1]_, tsv, + tsvz [#f2]_ + `pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, xlsm(read only) `xlwt`_ - `pyexcel-xlsx`_ xlsx `openpyxl`_ same as above - `pyexcel-ods3`_ ods `pyexcel-ezodf`_, 2.6, 2.7, 3.3, 3.4 - lxml 3.5, 3.6 - `pyexcel-ods`_ ods `odfpy`_ same as above - ======================== ======================= ================= ================== + `pyexcel-xlsx`_ xlsx `openpyxl`_ + `pyexcel-ods3`_ ods `pyexcel-ezodf`_, + lxml + `pyexcel-ods`_ ods `odfpy`_ + ======================== ======================= ================= .. table:: Dedicated file reader and writers - ======================== ======================= ================= ================== - Package name Supported file formats Dependencies Python versions - ======================== ======================= ================= ================== - `pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ Python 2 and 3 - `pyexcel-xlsxr`_ xlsx(read only) lxml same as above - `pyexcel-xlsbr`_ xlsx(read only) pyxlsb same as above - `pyexcel-odsr`_ read only for ods, fods lxml same as above - `pyexcel-odsw`_ write only for ods loxun same as above - `pyexcel-htmlr`_ html(read only) lxml,html5lib same as above - `pyexcel-pdfr`_ pdf(read only) pdftables Python 2 only. - ======================== ======================= ================= ================== + ======================== ======================= ================= + Package name Supported file formats Dependencies + ======================== ======================= ================= + `pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ + `pyexcel-libxlsxw`_ xlsx(write only) `libxlsxwriter`_ + `pyexcel-xlsxr`_ xlsx(read only) lxml + `pyexcel-xlsbr`_ xlsb(read only) pyxlsb + `pyexcel-odsr`_ read only for ods, fods lxml + `pyexcel-odsw`_ write only for ods loxun + `pyexcel-htmlr`_ html(read only) lxml,html5lib + `pyexcel-pdfr`_ pdf(read only) camelot + ======================== ======================= ================= Plugin shopping guide ------------------------ -Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of -xml files +Since 2020, all pyexcel-io plugins have dropped the support for python version +lower than 3.6. If you want to use any python verions, please use pyexcel-io +and its plugins version lower than 0.6.0. -The dedicated readers for excel files can stream read + +Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of +xml files + +The dedicated readers for excel files can stream read In order to manage the list of plugins installed, you need to use pip to add or remove @@ -133,6 +136,7 @@ You need to append get_array(..., library='pyexcel-odsr'). .. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr .. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw +.. _pyexcel-libxlsxw: https://github.com/pyexcel/pyexcel-libxlsxw .. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr .. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr .. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr @@ -143,6 +147,7 @@ You need to append get_array(..., library='pyexcel-odsr'). .. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter .. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf .. _odfpy: https://github.com/eea/odfpy +.. _libxlsxwriter: http://libxlsxwriter.github.io/getting_started.html .. rubric:: Footnotes diff --git a/docs/source/conf.py b/docs/source/conf.py index 9c84362..a2fc48f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'chfw' # The short X.Y version -version = '0.6.2' +version = '0.6.3' # The full version, including alpha/beta/rc tags release = '0.6.3' diff --git a/docs/source/index.rst b/docs/source/index.rst index 4b66b2c..6806a77 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,6 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License -:Development: |release| :Released: |version| :Generated: |today| @@ -65,44 +64,47 @@ For individual excel file formats, please install them as you wish: .. table:: A list of file formats supported by external plugins - ======================== ======================= ================= ================== - Package name Supported file formats Dependencies Python versions - ======================== ======================= ================= ================== - `pyexcel-io`_ >=v0.6.0 csv, csvz [#f1]_, tsv, 3.6+ - tsvz [#f2]_ - `pyexcel-io`_ <=0.5.20 same as above 2.6, 2.7, 3.3, - 3.4, 3.5, 3.6 - pypy - `pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, same as above + ======================== ======================= ================= + Package name Supported file formats Dependencies + ======================== ======================= ================= + `pyexcel-io`_ csv, csvz [#f1]_, tsv, + tsvz [#f2]_ + `pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, xlsm(read only) `xlwt`_ - `pyexcel-xlsx`_ xlsx `openpyxl`_ same as above - `pyexcel-ods3`_ ods `pyexcel-ezodf`_, 2.6, 2.7, 3.3, 3.4 - lxml 3.5, 3.6 - `pyexcel-ods`_ ods `odfpy`_ same as above - ======================== ======================= ================= ================== + `pyexcel-xlsx`_ xlsx `openpyxl`_ + `pyexcel-ods3`_ ods `pyexcel-ezodf`_, + lxml + `pyexcel-ods`_ ods `odfpy`_ + ======================== ======================= ================= .. table:: Dedicated file reader and writers - ======================== ======================= ================= ================== - Package name Supported file formats Dependencies Python versions - ======================== ======================= ================= ================== - `pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ Python 2 and 3 - `pyexcel-xlsxr`_ xlsx(read only) lxml same as above - `pyexcel-xlsbr`_ xlsx(read only) pyxlsb same as above - `pyexcel-odsr`_ read only for ods, fods lxml same as above - `pyexcel-odsw`_ write only for ods loxun same as above - `pyexcel-htmlr`_ html(read only) lxml,html5lib same as above - `pyexcel-pdfr`_ pdf(read only) pdftables Python 2 only. - ======================== ======================= ================= ================== + ======================== ======================= ================= + Package name Supported file formats Dependencies + ======================== ======================= ================= + `pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ + `pyexcel-libxlsxw`_ xlsx(write only) `libxlsxwriter`_ + `pyexcel-xlsxr`_ xlsx(read only) lxml + `pyexcel-xlsbr`_ xlsb(read only) pyxlsb + `pyexcel-odsr`_ read only for ods, fods lxml + `pyexcel-odsw`_ write only for ods loxun + `pyexcel-htmlr`_ html(read only) lxml,html5lib + `pyexcel-pdfr`_ pdf(read only) camelot + ======================== ======================= ================= Plugin shopping guide ------------------------ -Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of -xml files +Since 2020, all pyexcel-io plugins have dropped the support for python version +lower than 3.6. If you want to use any python verions, please use pyexcel-io +and its plugins version lower than 0.6.0. -The dedicated readers for excel files can stream read + +Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of +xml files + +The dedicated readers for excel files can stream read In order to manage the list of plugins installed, you need to use pip to add or remove @@ -124,6 +126,7 @@ You need to append get_array(..., library='pyexcel-odsr'). .. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr .. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw +.. _pyexcel-libxlsxw: https://github.com/pyexcel/pyexcel-libxlsxw .. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr .. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr .. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr @@ -134,6 +137,7 @@ You need to append get_array(..., library='pyexcel-odsr'). .. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter .. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf .. _odfpy: https://github.com/eea/odfpy +.. _libxlsxwriter: http://libxlsxwriter.github.io/getting_started.html .. rubric:: Footnotes diff --git a/lint.sh b/lint.sh index 891aa63..d31eeaa 100644 --- a/lint.sh +++ b/lint.sh @@ -1,2 +1,2 @@ pip install flake8 -flake8 --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long . && python setup.py checkdocs +flake8 --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long . && python setup.py checkdocs \ No newline at end of file diff --git a/setup.py b/setup.py index 6c9261b..4535574 100644 --- a/setup.py +++ b/setup.py @@ -79,19 +79,20 @@ SETUP_COMMANDS = {} PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests", "tests.*"]) EXTRAS_REQUIRE = { - "xls": ['pyexcel-xls>=0.5.0'], - "xlsx": ['pyexcel-xlsx>=0.5.0'], - "ods": ['pyexcel-ods3>=0.5.0'], + "xls": ['pyexcel-xls>=0.6.0'], + "xlsx": ['pyexcel-xlsx>=0.6.0'], + "ods": ['pyexcel-ods3>=0.6.0'], } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) +HERE = os.path.abspath(os.path.dirname(__file__)) + GS_COMMAND = ("gs pyexcel-io v0.6.3 " + "Find 0.6.3 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) -HERE = os.path.abspath(os.path.dirname(__file__)) class PublishCommand(Command): @@ -137,7 +138,6 @@ SETUP_COMMANDS.update({ "publish": PublishCommand }) - def has_gease(): """ test if github release command is installed From ab2f43987890fe0629953885f48a10038f860ae0 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 17 Oct 2020 22:35:01 +0100 Subject: [PATCH 113/143] :hammer: restore author's initials --- docs/source/conf.py | 2 +- docs/source/index.rst | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a2fc48f..4595678 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,7 +24,7 @@ DESCRIPTION = ( project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' -author = 'chfw' +author = 'C.W.' # The short X.Y version version = '0.6.3' # The full version, including alpha/beta/rc tags diff --git a/docs/source/index.rst b/docs/source/index.rst index 6806a77..00332e4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,7 +6,7 @@ `pyexcel-io` - Let you focus on data, instead of file formats ================================================================================ -:Author: chfw +:Author: C.W. :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License diff --git a/setup.py b/setup.py index 4535574..ec2f56d 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ except (ValueError, UnicodeError, locale.Error): locale.setlocale(locale.LC_ALL, "en_US.UTF-8") NAME = "pyexcel-io" -AUTHOR = "chfw" +AUTHOR = "C.W." VERSION = "0.6.3" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" From 04cfc27a0f21cc04d7863d9f45983a3f71d296c2 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 20 Oct 2020 23:18:46 +0100 Subject: [PATCH 115/143] :micrscope: test lml 0.1.0 --- README.rst | 4 +++- rnd_requirements.txt | 1 + setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 9ea17e3..4ec5d4e 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://raw.githubusercontent.com/pyexcel/pyexcel.github.io/master/images/patreon.png :target: https://www.patreon.com/chfw -.. image:: https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg +.. image:: https://raw.githubusercontent.com/pyexcel/pyexcel-mobans/master/images/awesome-badge.svg :target: https://awesome-python.com/#specific-formats-processing .. image:: https://travis-ci.org/pyexcel/pyexcel-io.svg?branch=master @@ -60,6 +60,8 @@ Known constraints Fonts, colors and charts are not supported. +Nor to read password protected xls, xlsx and ods files. + Introduction ================================================================================ diff --git a/rnd_requirements.txt b/rnd_requirements.txt index e69de29..ca02ee8 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -0,0 +1 @@ +https://github.com/python-lml/lml/archive/dev.zip diff --git a/setup.py b/setup.py index ec2f56d..967cc22 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ EXTRAS_REQUIRE = { PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) HERE = os.path.abspath(os.path.dirname(__file__)) -GS_COMMAND = ("gs pyexcel-io v0.6.3 " + +GS_COMMAND = ("gease pyexcel-io v0.6.3 " + "Find 0.6.3 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") From 4a0b0130f6634320ec1981b35c9d304a568aa032 Mon Sep 17 00:00:00 2001 From: jaska Date: Wed, 21 Oct 2020 22:45:35 +0100 Subject: [PATCH 116/143] Update rnd_requirements.txt --- rnd_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rnd_requirements.txt b/rnd_requirements.txt index ca02ee8..8b13789 100644 --- a/rnd_requirements.txt +++ b/rnd_requirements.txt @@ -1 +1 @@ -https://github.com/python-lml/lml/archive/dev.zip + From cd081dddaa7b7dce6c4338d705781fbe3e7d3c3d Mon Sep 17 00:00:00 2001 From: jaska Date: Thu, 29 Oct 2020 07:18:43 +0000 Subject: [PATCH 117/143] Django jump column (#103) * :sparkles: support skipping columns when importing a spreadsheet. fix #102 * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :books: update typo in changelog.yml * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst Co-authored-by: chfw --- CHANGELOG.rst | 8 ++++++++ README.rst | 2 +- changelog.yml | 6 ++++++ docs/source/conf.py | 2 +- docs/source/index.rst | 1 + pyexcel-io.yml | 4 ++-- pyexcel_io/database/common.py | 14 +++++++++----- pyexcel_io/database/importers/django.py | 9 ++++++++- pyexcel_io/database/importers/sqlalchemy.py | 15 +++++++++------ setup.py | 8 ++++---- tests/test_django_book.py | 18 +++++++++++++++--- 11 files changed, 64 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 92e656b..5289adb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.6.4 - tbd +-------------------------------------------------------------------------------- + +**updated** + +#. `#102 `_: skip columns from + imported excel sheet. + 0.6.3 - 12.10.2020 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 4ec5d4e..4dabd67 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ pyexcel-io - Let you focus on data, instead of file formats :target: https://anaconda.org/conda-forge/pyexcel-io .. image:: https://pepy.tech/badge/pyexcel-io/month - :target: https://pepy.tech/project/pyexcel-io/month + :target: https://pepy.tech/project/pyexcel-io .. image:: https://anaconda.org/conda-forge/pyexcel-io/badges/downloads.svg :target: https://anaconda.org/conda-forge/pyexcel-io diff --git a/changelog.yml b/changelog.yml index 47f4053..7c84efa 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - "`#102`: skip columns from imported excel sheet." + version: 0.6.4 + date: tbd - changes: - action: fixed details: diff --git a/docs/source/conf.py b/docs/source/conf.py index 4595678..115126c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ author = 'C.W.' # The short X.Y version version = '0.6.3' # The full version, including alpha/beta/rc tags -release = '0.6.3' +release = '0.6.4' # -- General configuration --------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 00332e4..ad3a030 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License +:Development: |release| :Released: |version| :Generated: |today| diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 8b43aff..a4a3683 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -3,8 +3,8 @@ project: "pyexcel-io" name: pyexcel-io nick_name: io version: 0.6.3 -current_version: 0.6.3 -release: 0.6.3 +current_version: 0.6.4 +release: 0.6.4 copyright_year: 2015-2020 moban_command: false is_on_conda: true diff --git a/pyexcel_io/database/common.py b/pyexcel_io/database/common.py index 1f60ebd..a4a1994 100644 --- a/pyexcel_io/database/common.py +++ b/pyexcel_io/database/common.py @@ -86,11 +86,15 @@ class DjangoModelImportAdapter(DjangoModelExportAdapter): self.__column_name_mapping_dict.output = None elif isinstance(self.__column_name_mapping_dict.input, dict): if self.__column_names.input: - self.__column_names.output = [ - self.__column_name_mapping_dict.input[name] - for name in self.__column_names.input - ] - self.__column_name_mapping_dict.output = None + self.__column_names.output = [] + indices = [] + for index, name in enumerate(self.__column_names.input): + if name in self.__column_name_mapping_dict.input: + self.__column_names.output.append( + self.__column_name_mapping_dict.input[name] + ) + indices.append(index) + self.__column_name_mapping_dict.output = indices if self.__column_names.output is None: self.__column_names.output = self.__column_names.input diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 9d7307e..78592cf 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -33,13 +33,20 @@ class DjangoModelWriter(ISheetWriter): print(constants.MESSAGE_EMPTY_ARRAY) else: new_array = swap_empty_string_for_none(array) + if self.__mapdict: + another_new_array = [] + for index, element in enumerate(new_array): + if index in self.__mapdict: + another_new_array.append(element) + new_array = another_new_array model_to_be_created = new_array if self.__initializer is not None: model_to_be_created = self.__initializer(new_array) if model_to_be_created: + row = dict(zip(self.__column_names, model_to_be_created)) self.__objs.append( self.__model( - **dict(zip(self.__column_names, model_to_be_created)) + **row ) ) diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 5c5cd27..cf2705a 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -45,7 +45,14 @@ class SQLTableWriter(ISheetWriter): print(new_array) def _write_row(self, array): - row = dict(zip(self.adapter.column_names, array)) + new_array = array + if self.adapter.column_name_mapping_dict: + another_new_array = [] + for index, element in enumerate(new_array): + if index in self.adapter.column_name_mapping_dict: + another_new_array.append(element) + new_array = another_new_array + row = dict(zip(self.adapter.column_names, new_array)) obj = None if self.adapter.row_initializer: # allow initinalizer to return None @@ -54,11 +61,7 @@ class SQLTableWriter(ISheetWriter): if obj is None: obj = self.adapter.table() for name in self.adapter.column_names: - if self.adapter.column_name_mapping_dict is not None: - key = self.adapter.column_name_mapping_dict[name] - else: - key = name - setattr(obj, key, row[name]) + setattr(obj, name, row[name]) self.importer.session.add(obj) if self.__auto_commit and self.__bulk_size != float("inf"): self.__count += 1 diff --git a/setup.py b/setup.py index 967cc22..267d6a2 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "C.W." -VERSION = "0.6.3" +VERSION = "0.6.4" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( @@ -40,7 +40,7 @@ DESCRIPTION = ( "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.6.3.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.4.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -87,8 +87,8 @@ EXTRAS_REQUIRE = { PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) HERE = os.path.abspath(os.path.dirname(__file__)) -GS_COMMAND = ("gease pyexcel-io v0.6.3 " + - "Find 0.6.3 in changelog for more details") +GS_COMMAND = ("gease pyexcel-io v0.6.4 " + + "Find 0.6.4 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( diff --git a/tests/test_django_book.py b/tests/test_django_book.py index 5db3e7c..a95d7ab 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -158,10 +158,10 @@ class TestSheet: writer = DjangoModelWriter(None, adapter) writer.write_array(self.data[1:]) writer.close() - assert model.objects.objs == [ + eq_(model.objects.objs, [ {"Y": 2, "X": 2, "Z": 3}, {"Y": 5, "X": 5, "Z": 6}, - ] + ]) def test_sheet_save_to_django_model_skip_me(self): model = FakeDjangoModel() @@ -178,7 +178,7 @@ class TestSheet: writer = DjangoModelWriter(None, adapter) writer.write_array(self.data[1:]) writer.close() - assert model.objects.objs == [{"Y": 2, "X": 1, "Z": 3}] + eq_(model.objects.objs, [{"Y": 2, "X": 1, "Z": 3}]) def test_load_sheet_from_django_model(self): model = FakeDjangoModel() @@ -241,6 +241,18 @@ class TestSheet: writer.close() eq_(model.objects.objs, self.result) + def test_jumping_columns(self): + data2 = [["D", "A", "B", "C"], [1, 1, 2, 3], [10, 4, 5, 6]] + mapdict = {"C": "Z", "A": "X", "B": "Y"} + model = FakeDjangoModel() + adapter = DjangoModelImportAdapter(model) + adapter.column_names = data2[0] + adapter.column_name_mapping_dict = mapdict + writer = DjangoModelWriter(None, adapter) + writer.write_array(data2[1:]) + writer.close() + eq_(model.objects.objs, self.result) + def test_empty_model(self): model = FakeDjangoModel() reader = DjangoModelReader(model) From 783bbb5acc33116297915bc93b75d435863d49e8 Mon Sep 17 00:00:00 2001 From: jaska Date: Sat, 31 Oct 2020 11:30:18 +0000 Subject: [PATCH 118/143] Django plug-in Enhancement (#105) * :lipstick: remove double underscores on private variables. it sucks when debugging * :books: update change log * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst Co-authored-by: chfw --- CHANGELOG.rst | 2 +- changelog.yml | 2 +- docs/source/conf.py | 2 +- docs/source/index.rst | 1 - pyexcel-io.yml | 2 +- pyexcel_io/database/common.py | 57 +++++++++++++------------ pyexcel_io/database/importers/django.py | 43 ++++++++++--------- 7 files changed, 55 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5289adb..9493eb4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Change log ================================================================================ -0.6.4 - tbd +0.6.4 - 31.10.2020 -------------------------------------------------------------------------------- **updated** diff --git a/changelog.yml b/changelog.yml index 7c84efa..2716ee5 100644 --- a/changelog.yml +++ b/changelog.yml @@ -6,7 +6,7 @@ releases: details: - "`#102`: skip columns from imported excel sheet." version: 0.6.4 - date: tbd + date: 31.10.2020 - changes: - action: fixed details: diff --git a/docs/source/conf.py b/docs/source/conf.py index 115126c..de27cdb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'C.W.' # The short X.Y version -version = '0.6.3' +version = '0.6.4' # The full version, including alpha/beta/rc tags release = '0.6.4' diff --git a/docs/source/index.rst b/docs/source/index.rst index ad3a030..00332e4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,6 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License -:Development: |release| :Released: |version| :Generated: |today| diff --git a/pyexcel-io.yml b/pyexcel-io.yml index a4a3683..fbf944e 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,7 +2,7 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.3 +version: 0.6.4 current_version: 0.6.4 release: 0.6.4 copyright_year: 2015-2020 diff --git a/pyexcel_io/database/common.py b/pyexcel_io/database/common.py index a4a1994..f074609 100644 --- a/pyexcel_io/database/common.py +++ b/pyexcel_io/database/common.py @@ -38,65 +38,66 @@ class DjangoModelImportAdapter(DjangoModelExportAdapter): def __init__(self, model): DjangoModelExportAdapter.__init__(self, model) - self.__column_names = self.InOutParameter() - self.__column_name_mapping_dict = self.InOutParameter() - self.__row_initializer = self.InOutParameter() + self._column_names = self.InOutParameter() + self._column_name_mapping_dict = self.InOutParameter() + self._row_initializer = self.InOutParameter() self._process_parameters() @property def row_initializer(self): """ contructor for a database table entry """ - return self.__row_initializer.output + return self._row_initializer.output @property def column_names(self): """ the desginated database column names """ - return self.__column_names.output + return self._column_names.output @property def column_name_mapping_dict(self): """ if not the same, a mapping dictionary is looked up""" - return self.__column_name_mapping_dict.output + return self._column_name_mapping_dict.output @row_initializer.setter def row_initializer(self, a_function): """ set the contructor """ - self.__row_initializer.input = a_function + self._row_initializer.input = a_function self._process_parameters() @column_names.setter def column_names(self, column_names): """ set the column names """ - self.__column_names.input = column_names + self._column_names.input = column_names self._process_parameters() @column_name_mapping_dict.setter def column_name_mapping_dict(self, mapping_dict): """ set the mapping dict """ - self.__column_name_mapping_dict.input = mapping_dict + self._column_name_mapping_dict.input = mapping_dict self._process_parameters() def _process_parameters(self): - if self.__row_initializer.input is None: - self.__row_initializer.output = None + if self._row_initializer.input is None: + self._row_initializer.output = None else: - self.__row_initializer.output = self.__row_initializer.input - if isinstance(self.__column_name_mapping_dict.input, list): - self.__column_names.output = self.__column_name_mapping_dict.input - self.__column_name_mapping_dict.output = None - elif isinstance(self.__column_name_mapping_dict.input, dict): - if self.__column_names.input: - self.__column_names.output = [] + self._row_initializer.output = self._row_initializer.input + if isinstance(self._column_name_mapping_dict.input, list): + self._column_names.output = self._column_name_mapping_dict.input + self._column_name_mapping_dict.output = None + elif isinstance(self._column_name_mapping_dict.input, dict): + + if self._column_names.input: + self._column_names.output = [] indices = [] - for index, name in enumerate(self.__column_names.input): - if name in self.__column_name_mapping_dict.input: - self.__column_names.output.append( - self.__column_name_mapping_dict.input[name] + for index, name in enumerate(self._column_names.input): + if name in self._column_name_mapping_dict.input: + self._column_names.output.append( + self._column_name_mapping_dict.input[name] ) indices.append(index) - self.__column_name_mapping_dict.output = indices - if self.__column_names.output is None: - self.__column_names.output = self.__column_names.input + self._column_name_mapping_dict.output = indices + if self._column_names.output is None: + self._column_names.output = self._column_names.input class DjangoModelExporter(object): @@ -114,15 +115,15 @@ class DjangoModelImporter(object): """ public interface for django model import """ def __init__(self): - self.__adapters = {} + self._adapters = {} def append(self, import_adapter): """ store model parameter for more than one model """ - self.__adapters[import_adapter.get_name()] = import_adapter + self._adapters[import_adapter.get_name()] = import_adapter def get(self, name): """ get a parameter out """ - return self.__adapters.get(name, None) + return self._adapters.get(name, None) class SQLTableExportAdapter(DjangoModelExportAdapter): diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 78592cf..0904287 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -20,32 +20,33 @@ class DjangoModelWriter(ISheetWriter): """ import data into a django model """ def __init__(self, importer, adapter, batch_size=None, bulk_save=True): - self.__batch_size = batch_size - self.__model = adapter.model - self.__column_names = adapter.column_names - self.__mapdict = adapter.column_name_mapping_dict - self.__initializer = adapter.row_initializer - self.__objs = [] - self.__bulk_save = bulk_save + self.batch_size = batch_size + self.model = adapter.model + self.column_names = adapter.column_names + self.mapdict = adapter.column_name_mapping_dict + self.initializer = adapter.row_initializer + self.objs = [] + self.bulk_save = bulk_save + self.adapter = adapter def write_row(self, array): if is_empty_array(array): print(constants.MESSAGE_EMPTY_ARRAY) else: new_array = swap_empty_string_for_none(array) - if self.__mapdict: + if self.mapdict: another_new_array = [] for index, element in enumerate(new_array): - if index in self.__mapdict: + if index in self.mapdict: another_new_array.append(element) new_array = another_new_array model_to_be_created = new_array - if self.__initializer is not None: - model_to_be_created = self.__initializer(new_array) + if self.initializer is not None: + model_to_be_created = self.initializer(new_array) if model_to_be_created: - row = dict(zip(self.__column_names, model_to_be_created)) - self.__objs.append( - self.__model( + row = dict(zip(self.column_names, model_to_be_created)) + self.objs.append( + self.model( **row ) ) @@ -54,12 +55,12 @@ class DjangoModelWriter(ISheetWriter): # skip the row def close(self): - if self.__bulk_save: - self.__model.objects.bulk_create( - self.__objs, batch_size=self.__batch_size + if self.bulk_save: + self.model.objects.bulk_create( + self.objs, batch_size=self.batch_size ) else: - for an_object in self.__objs: + for an_object in self.objs: an_object.save() @@ -67,15 +68,15 @@ class DjangoBookWriter(IWriter): """ write data into django models """ def __init__(self, exporter, _, **keywords): - self.__importer = exporter + self.importer = exporter self._keywords = keywords def create_sheet(self, sheet_name): sheet_writer = None - model = self.__importer.get(sheet_name) + model = self.importer.get(sheet_name) if model: sheet_writer = DjangoModelWriter( - self.__importer, + self.importer, model, batch_size=self._keywords.get("batch_size", None), bulk_save=self._keywords.get("bulk_save", True), From 98e0e762b6c6f90e126d7b1e647bf1b921dd68a8 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 31 Oct 2020 12:36:54 +0000 Subject: [PATCH 119/143] :lipstick: update coding style --- pyexcel_io/database/importers/django.py | 6 +----- tests/test_django_book.py | 11 +++++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 0904287..49ee14c 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -45,11 +45,7 @@ class DjangoModelWriter(ISheetWriter): model_to_be_created = self.initializer(new_array) if model_to_be_created: row = dict(zip(self.column_names, model_to_be_created)) - self.objs.append( - self.model( - **row - ) - ) + self.objs.append(self.model(**row)) # else # skip the row diff --git a/tests/test_django_book.py b/tests/test_django_book.py index a95d7ab..5ccdd01 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -158,10 +158,13 @@ class TestSheet: writer = DjangoModelWriter(None, adapter) writer.write_array(self.data[1:]) writer.close() - eq_(model.objects.objs, [ - {"Y": 2, "X": 2, "Z": 3}, - {"Y": 5, "X": 5, "Z": 6}, - ]) + eq_( + model.objects.objs, + [ + {"Y": 2, "X": 2, "Z": 3}, + {"Y": 5, "X": 5, "Z": 6}, + ], + ) def test_sheet_save_to_django_model_skip_me(self): model = FakeDjangoModel() From c6111784be97d88ea803edbc5f4a9050dcfd0647 Mon Sep 17 00:00:00 2001 From: jaska Date: Sat, 28 Nov 2020 18:53:18 +0000 Subject: [PATCH 120/143] windows bulid on github actions (#107) * :sparkles: enable github actions for pyexcel-io. https://github.com/pyexcel/pyexcel/issues/238, https://github.com/pyexcel/pyexcel * :bug: in test. memory map file on windows needs close. * :green_heart: increase verbosity * :green_heart: setup utf-8 encoding * :green_heart: give a line break * :green_heart: verify the tests on windows * :green_heart: python 3 need no unicode annotation * :green_heart: use Finnish world instead of Chinese one for now * :fire: extra statement had no use * :fire: remove travis config * :fire: repalce travis status * :sparkles: kick start next version * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst Co-authored-by: chfw --- .github/PULL_REQUEST_TEMPLATE.md | 1 - .github/workflows/lint.yml | 20 ++++++++++++++ .github/workflows/tests.yml | 32 +++++++++++++++++++++++ .moban.yml | 1 - .travis.yml | 45 -------------------------------- README.rst | 4 +-- docs/source/conf.py | 2 +- docs/source/index.rst | 1 + docs/source/plaincsv.rst | 2 +- pyexcel-io.yml | 4 +-- setup.py | 2 +- tests/test_issues.py | 1 + 12 files changed, 61 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/tests.yml delete mode 100644 .travis.yml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6017f21..8996445 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,6 +4,5 @@ With your PR, here is a check list: - [ ] Has all code lines tested? - [ ] Has `make format` been run? - [ ] Please update CHANGELOG.yml(not CHANGELOG.rst) -- [ ] Passes all Travis CI builds - [ ] Has fair amount of documentation if your change is complex - [ ] Agree on NEW BSD License for your contribution diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..03122a1 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,20 @@ +name: lint + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + name: lint code + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: lint + run: | + pip install flake8 + pip install -r tests/requirements.txt + flake8 --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long . + python setup.py checkdocs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..5703185 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,32 @@ +name: run_tests + +on: [push, pull_request] + +jobs: + test: + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + os: [macOs-latest, ubuntu-latest, windows-latest] + + runs-on: ${{ matrix.os }} + name: run tests + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: install + run: | + pip install -r requirements.txt + pip install -r tests/requirements.txt + - name: test + run: | + pip freeze + nosetests --verbosity=3 --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io + - name: Upload coverage + uses: codecov/codecov-action@v1 + with: + name: ${{ matrix.os }} Python ${{ matrix.python-version }} \ No newline at end of file diff --git a/.moban.yml b/.moban.yml index b73e1fa..864d61c 100644 --- a/.moban.yml +++ b/.moban.yml @@ -4,7 +4,6 @@ configuration: targets: - "docs/source/conf.py": "docs/source/custom_conf.py.jj2" - setup.py: io_setup.py.jj2 - - .travis.yml: custom_travis.yml.jj2 - README.rst: io_readme.rst.jj2 - "docs/source/index.rst": "docs/source/index.rst.jj2" - .gitignore: gitignore.jj2 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 208ab50..0000000 --- a/.travis.yml +++ /dev/null @@ -1,45 +0,0 @@ -sudo: false -dist: xenial -language: python -notifications: - email: false -python: - - 3.8 - - 3.7 - - 3.6 -env: - - MINREQ=1 - -stages: - - lint - - test - - -.lint: &lint - git: - submodules: false - python: 3.6 - env: - - MINREQ=0 - stage: lint - script: make lint - -jobs: - include: - - *moban - - *lint - -stage: test - -before_install: - - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then - mv min_requirements.txt requirements.txt ; - fi - - test ! -f rnd_requirements.txt || - pip install --no-deps -r rnd_requirements.txt - - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; - - pip install -r tests/requirements.txt -script: - - make test -after_success: - codecov diff --git a/README.rst b/README.rst index 4dabd67..4a3e5ae 100644 --- a/README.rst +++ b/README.rst @@ -8,8 +8,8 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://raw.githubusercontent.com/pyexcel/pyexcel-mobans/master/images/awesome-badge.svg :target: https://awesome-python.com/#specific-formats-processing -.. image:: https://travis-ci.org/pyexcel/pyexcel-io.svg?branch=master - :target: http://travis-ci.org/pyexcel/pyexcel-io +.. image:: https://github.com/actions/pyexcel/pyexcel-io/workflows/.github/workflows/tests.yml/badge.svg + :target: http://github.com/pyexcel/pyexcel-io .. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg :target: https://codecov.io/gh/pyexcel/pyexcel-io diff --git a/docs/source/conf.py b/docs/source/conf.py index de27cdb..5da08a2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'C.W.' # The short X.Y version -version = '0.6.4' +version = '0.6.5' # The full version, including alpha/beta/rc tags release = '0.6.4' diff --git a/docs/source/index.rst b/docs/source/index.rst index 00332e4..ad3a030 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License +:Development: |release| :Released: |version| :Generated: |today| diff --git a/docs/source/plaincsv.rst b/docs/source/plaincsv.rst index c6c3490..c009627 100644 --- a/docs/source/plaincsv.rst +++ b/docs/source/plaincsv.rst @@ -153,7 +153,7 @@ Here is an example to write a sentence of "Shui Dial Getou"[#f2] into a csv file .. code-block:: python - >>> content = [[u'人有悲歡離合', u'月有陰晴圓缺']] + >>> content = [['löyly', 'löyly']] >>> test_file = "test-utf8-BOM.csv" >>> save_data(test_file, content, encoding="utf-8-sig", lineterminator="\n") diff --git a/pyexcel-io.yml b/pyexcel-io.yml index fbf944e..2ba98b1 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,8 +2,8 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.4 -current_version: 0.6.4 +version: 0.6.5 +current_version: 0.6.5 release: 0.6.4 copyright_year: 2015-2020 moban_command: false diff --git a/setup.py b/setup.py index 267d6a2..4bad324 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "C.W." -VERSION = "0.6.4" +VERSION = "0.6.5" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( diff --git a/tests/test_issues.py b/tests/test_issues.py index d9cbfd6..6486c6a 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -103,6 +103,7 @@ def check_mmap_encoding(encoding): memory_mapped_file = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) data = get_data(memory_mapped_file, file_type="csv", encoding=encoding) eq_(data["csv"], content) + memory_mapped_file.close() os.unlink(test_file) From 6568017a8181a252e7e2c22f7afdb3fdb4b97d82 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 28 Nov 2020 18:55:37 +0000 Subject: [PATCH 121/143] :books: update github action link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4a3e5ae..2eb2e5f 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ pyexcel-io - Let you focus on data, instead of file formats :target: https://awesome-python.com/#specific-formats-processing .. image:: https://github.com/actions/pyexcel/pyexcel-io/workflows/.github/workflows/tests.yml/badge.svg - :target: http://github.com/pyexcel/pyexcel-io + :target: http://github.com/pyexcel/pyexcel-io/actions .. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg :target: https://codecov.io/gh/pyexcel/pyexcel-io From d91c32ffb1aac50d94d075122890e3e555a054da Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 28 Nov 2020 22:12:08 +0000 Subject: [PATCH 122/143] :books: update run tests status badge --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2eb2e5f..a289f87 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ pyexcel-io - Let you focus on data, instead of file formats .. image:: https://raw.githubusercontent.com/pyexcel/pyexcel-mobans/master/images/awesome-badge.svg :target: https://awesome-python.com/#specific-formats-processing -.. image:: https://github.com/actions/pyexcel/pyexcel-io/workflows/.github/workflows/tests.yml/badge.svg +.. image:: https://github.com/pyexcel/pyexcel-io/workflows/run_tests/badge.svg :target: http://github.com/pyexcel/pyexcel-io/actions .. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg From 5e74c8b666e2c5cf645335fe3babd58004ad8576 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 5 Dec 2020 22:20:21 +0000 Subject: [PATCH 123/143] :sparkles: configure the readthedocs using yml file --- .readthedocs.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..2d3c7f9 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,20 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optionally build your docs in additional formats such as PDF +formats: + - pdf + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.7 + install: + - requirements: docs/requirements.txt From 9ebfa3c79520a5ea8fb5ede7134e494d1b8145d8 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 5 Dec 2020 22:31:22 +0000 Subject: [PATCH 124/143] :fire: remove custom requirements.txt --- .readthedocs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 2d3c7f9..a379070 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,5 +16,3 @@ formats: # Optionally set the version of Python and requirements required to build your docs python: version: 3.7 - install: - - requirements: docs/requirements.txt From 66906c4e230bffe063e3e5e6c732dea929107ec9 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 5 Dec 2020 22:43:12 +0000 Subject: [PATCH 125/143] :books: update next version --- pyexcel-io.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 2ba98b1..5a199bf 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -3,7 +3,7 @@ project: "pyexcel-io" name: pyexcel-io nick_name: io version: 0.6.5 -current_version: 0.6.5 +current_version: 0.6.4 release: 0.6.4 copyright_year: 2015-2020 moban_command: false diff --git a/setup.py b/setup.py index 4bad324..267d6a2 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "C.W." -VERSION = "0.6.5" +VERSION = "0.6.4" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( From 67c111f0761775b3bba914387007833a1430e405 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 5 Dec 2020 22:49:10 +0000 Subject: [PATCH 126/143] :books: update next version --- docs/source/conf.py | 4 ++-- pyexcel-io.yml | 6 +++--- setup.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 5da08a2..a830e40 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,9 +26,9 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'C.W.' # The short X.Y version -version = '0.6.5' +version = '0.6.4' # The full version, including alpha/beta/rc tags -release = '0.6.4' +release = '0.6.5' # -- General configuration --------------------------------------------------- diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 5a199bf..06bc073 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,9 +2,9 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.5 -current_version: 0.6.4 -release: 0.6.4 +version: 0.6.4 +current_version: 0.6.5 +release: 0.6.5 copyright_year: 2015-2020 moban_command: false is_on_conda: true diff --git a/setup.py b/setup.py index 267d6a2..ba6cd9e 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "C.W." -VERSION = "0.6.4" +VERSION = "0.6.5" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( @@ -40,7 +40,7 @@ DESCRIPTION = ( "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.6.4.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.5.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -87,8 +87,8 @@ EXTRAS_REQUIRE = { PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) HERE = os.path.abspath(os.path.dirname(__file__)) -GS_COMMAND = ("gease pyexcel-io v0.6.4 " + - "Find 0.6.4 in changelog for more details") +GS_COMMAND = ("gease pyexcel-io v0.6.5 " + + "Find 0.6.5 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( From 04f992c2edfb864e3391e60c6c74303777db3280 Mon Sep 17 00:00:00 2001 From: vinraspa <78381185+vinraspa@users.noreply.github.com> Date: Fri, 1 Oct 2021 23:34:35 +0200 Subject: [PATCH 127/143] Update service.py (#109) Add datetime.datetime: "datetime" to ODS_WRITE_FORMAT_COVERSION dict --- pyexcel_io/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyexcel_io/service.py b/pyexcel_io/service.py index dcc92de..02d0a0e 100644 --- a/pyexcel_io/service.py +++ b/pyexcel_io/service.py @@ -173,6 +173,7 @@ ODS_WRITE_FORMAT_COVERSION = { datetime.date: "date", datetime.time: "time", datetime.timedelta: "timedelta", + datetime.datetime: "datetime", bool: "boolean", } From c8102bd63891a3ecaf9c38d0e94e4056e0474d2c Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 7 Oct 2021 21:51:24 +0100 Subject: [PATCH 128/143] :handshake: update project static information --- .github/workflows/lint.yml | 4 ++-- .github/workflows/moban-update.yml | 7 +++---- .github/workflows/tests.yml | 7 +++++-- CONTRIBUTORS.rst | 3 ++- README.rst | 12 ++++++------ docs/source/index.rst | 6 +++--- 6 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 03122a1..3789494 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: python-version: 3.8 - name: lint run: | - pip install flake8 - pip install -r tests/requirements.txt + pip --use-deprecated=legacy-resolver install flake8 + pip --use-deprecated=legacy-resolver install -r tests/requirements.txt flake8 --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long . python setup.py checkdocs diff --git a/.github/workflows/moban-update.yml b/.github/workflows/moban-update.yml index 706fd82..73a3aed 100644 --- a/.github/workflows/moban-update.yml +++ b/.github/workflows/moban-update.yml @@ -8,6 +8,7 @@ jobs: - uses: actions/checkout@v2 with: ref: ${{ github.head_ref }} + token: ${{ secrets.PAT }} - name: Set up Python uses: actions/setup-python@v1 with: @@ -20,10 +21,8 @@ jobs: git diff --exit-code - name: Auto-commit if: failure() - uses: docker://cdssnc/auto-commit-github-action - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: stefanzweifel/git-auto-commit-action@v4 with: - args: >- + commit_message: >- This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5703185..810f878 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,9 @@ jobs: matrix: python-version: [3.6, 3.7, 3.8, 3.9] os: [macOs-latest, ubuntu-latest, windows-latest] + exclude: + - os: macOs-latest + python-version: 3.6 runs-on: ${{ matrix.os }} name: run tests @@ -20,8 +23,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: install run: | - pip install -r requirements.txt - pip install -r tests/requirements.txt + pip --use-deprecated=legacy-resolver install -r requirements.txt + pip --use-deprecated=legacy-resolver install -r tests/requirements.txt - name: test run: | pip freeze diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index ba2e01d..5a6c33d 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -1,6 +1,6 @@ -5 contributors +6 contributors ================================================================================ In alphabetical order: @@ -9,4 +9,5 @@ In alphabetical order: * `John Vandenberg `_ * `Stephen J. Fuhry `_ * `Stephen Rauch `_ +* `vinraspa `_ * `Víctor Antonio Hernández Monroy `_ diff --git a/README.rst b/README.rst index a289f87..c4f43b7 100644 --- a/README.rst +++ b/README.rst @@ -108,9 +108,9 @@ sqlalchemy supported databases. Its supported file formats are extended to cover Plugin shopping guide ------------------------ -Since 2020, all pyexcel-io plugins have dropped the support for python version -lower than 3.6. If you want to use any python verions, please use pyexcel-io -and its plugins version lower than 0.6.0. +Since 2020, all pyexcel-io plugins have dropped the support for python versions +which are lower than 3.6. If you want to use any of those Python versions, please use pyexcel-io +and its plugins versions that are lower than 0.6.0. Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of @@ -205,7 +205,7 @@ Then install relevant development requirements: #. pip install -r tests/requirements.txt Once you have finished your changes, please provide test case(s), relevant documentation -and update CHANGELOG.rst. +and update changelog.yml .. note:: @@ -224,7 +224,7 @@ On Linux/Unix systems, please launch your tests like this:: $ make -On Windows systems, please issue this command:: +On Windows, please issue this command:: > test.bat @@ -236,7 +236,7 @@ Please run:: $ make format -so as to beautify your code otherwise travis-ci may fail your unit test. +so as to beautify your code otherwise your build may fail your unit test. diff --git a/docs/source/index.rst b/docs/source/index.rst index ad3a030..ac12734 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -97,9 +97,9 @@ For individual excel file formats, please install them as you wish: Plugin shopping guide ------------------------ -Since 2020, all pyexcel-io plugins have dropped the support for python version -lower than 3.6. If you want to use any python verions, please use pyexcel-io -and its plugins version lower than 0.6.0. +Since 2020, all pyexcel-io plugins have dropped the support for python versions +which are lower than 3.6. If you want to use any of those Python versions, please use pyexcel-io +and its plugins versions that are lower than 0.6.0. Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of From e40737495bbc4b382b567911c5d6a07670a9af03 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 8 Oct 2021 08:05:37 +0100 Subject: [PATCH 129/143] :lipstick: use latest black formatter --- pyexcel_io/_compact.py | 2 +- pyexcel_io/database/common.py | 40 ++++++++++----------- pyexcel_io/database/exporters/django.py | 2 +- pyexcel_io/database/exporters/sqlalchemy.py | 2 +- pyexcel_io/database/importers/django.py | 4 +-- pyexcel_io/database/importers/sqlalchemy.py | 2 +- pyexcel_io/database/querysets.py | 6 ++-- pyexcel_io/plugins.py | 20 +++++------ pyexcel_io/readers/csv_sheet.py | 8 ++--- pyexcel_io/utils.py | 4 +-- pyexcel_io/writers/csv_sheet.py | 2 +- pyexcel_io/writers/csvz_sheet.py | 2 +- 12 files changed, 47 insertions(+), 47 deletions(-) diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index 1e0fa18..dd75146 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -27,7 +27,7 @@ PY2 = sys.version[0] == 2 def isstream(instance): - """ check if a instance is a stream """ + """check if a instance is a stream""" try: import mmap diff --git a/pyexcel_io/database/common.py b/pyexcel_io/database/common.py index f074609..1008c1b 100644 --- a/pyexcel_io/database/common.py +++ b/pyexcel_io/database/common.py @@ -10,7 +10,7 @@ class DjangoModelExportAdapter(object): - """ django export parameter holder """ + """django export parameter holder""" def __init__(self, model, export_columns=None): self.model = model @@ -18,19 +18,19 @@ class DjangoModelExportAdapter(object): @property def name(self): - """ get database table name """ + """get database table name""" return self.get_name() def get_name(self): - """ get database table name """ + """get database table name""" return self.model._meta.model_name class DjangoModelImportAdapter(DjangoModelExportAdapter): - """ parameter holder for django data import """ + """parameter holder for django data import""" class InOutParameter(object): - """ local class to manipulate variable io """ + """local class to manipulate variable io""" def __init__(self): self.output = None @@ -45,34 +45,34 @@ class DjangoModelImportAdapter(DjangoModelExportAdapter): @property def row_initializer(self): - """ contructor for a database table entry """ + """contructor for a database table entry""" return self._row_initializer.output @property def column_names(self): - """ the desginated database column names """ + """the desginated database column names""" return self._column_names.output @property def column_name_mapping_dict(self): - """ if not the same, a mapping dictionary is looked up""" + """if not the same, a mapping dictionary is looked up""" return self._column_name_mapping_dict.output @row_initializer.setter def row_initializer(self, a_function): - """ set the contructor """ + """set the contructor""" self._row_initializer.input = a_function self._process_parameters() @column_names.setter def column_names(self, column_names): - """ set the column names """ + """set the column names""" self._column_names.input = column_names self._process_parameters() @column_name_mapping_dict.setter def column_name_mapping_dict(self, mapping_dict): - """ set the mapping dict """ + """set the mapping dict""" self._column_name_mapping_dict.input = mapping_dict self._process_parameters() @@ -101,33 +101,33 @@ class DjangoModelImportAdapter(DjangoModelExportAdapter): class DjangoModelExporter(object): - """ public interface for django model export """ + """public interface for django model export""" def __init__(self): self.adapters = [] def append(self, import_adapter): - """ store model parameter for more than one model """ + """store model parameter for more than one model""" self.adapters.append(import_adapter) class DjangoModelImporter(object): - """ public interface for django model import """ + """public interface for django model import""" def __init__(self): self._adapters = {} def append(self, import_adapter): - """ store model parameter for more than one model """ + """store model parameter for more than one model""" self._adapters[import_adapter.get_name()] = import_adapter def get(self, name): - """ get a parameter out """ + """get a parameter out""" return self._adapters.get(name, None) class SQLTableExportAdapter(DjangoModelExportAdapter): - """ parameter holder for sql table data export """ + """parameter holder for sql table data export""" def __init__(self, model, export_columns=None): DjangoModelExportAdapter.__init__(self, model, export_columns) @@ -138,7 +138,7 @@ class SQLTableExportAdapter(DjangoModelExportAdapter): class SQLTableImportAdapter(DjangoModelImportAdapter): - """ parameter holder for sqlalchemy table import """ + """parameter holder for sqlalchemy table import""" def __init__(self, model): DjangoModelImportAdapter.__init__(self, model) @@ -149,7 +149,7 @@ class SQLTableImportAdapter(DjangoModelImportAdapter): class SQLTableExporter(DjangoModelExporter): - """ public interface for sql table export """ + """public interface for sql table export""" def __init__(self, session): DjangoModelExporter.__init__(self) @@ -157,7 +157,7 @@ class SQLTableExporter(DjangoModelExporter): class SQLTableImporter(DjangoModelImporter): - """ public interface to do data import via sqlalchemy """ + """public interface to do data import via sqlalchemy""" def __init__(self, session): DjangoModelImporter.__init__(self) diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index d642c5f..018481a 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -28,7 +28,7 @@ class DjangoModelReader(QuerysetsReader): class DjangoBookReader(IReader): - """ read django models """ + """read django models""" def __init__(self, exporter, _, **keywords): self.exporter = exporter diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index 1e084b8..dbdc4c7 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -32,7 +32,7 @@ class SQLTableReader(QuerysetsReader): class SQLBookReader(IReader): - """ read a table via sqlalchemy """ + """read a table via sqlalchemy""" def __init__(self, exporter, _, **keywords): self.__exporter = exporter diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 49ee14c..1c30f3d 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -17,7 +17,7 @@ log = logging.getLogger(__name__) class DjangoModelWriter(ISheetWriter): - """ import data into a django model """ + """import data into a django model""" def __init__(self, importer, adapter, batch_size=None, bulk_save=True): self.batch_size = batch_size @@ -61,7 +61,7 @@ class DjangoModelWriter(ISheetWriter): class DjangoBookWriter(IWriter): - """ write data into django models """ + """write data into django models""" def __init__(self, exporter, _, **keywords): self.importer = exporter diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index cf2705a..2cb0c30 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -74,7 +74,7 @@ class SQLTableWriter(ISheetWriter): class SQLBookWriter(IWriter): - """ write data into database tables via sqlalchemy """ + """write data into database tables via sqlalchemy""" def __init__(self, file_content, _, auto_commit=True, **keywords): self.__importer = file_content diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index d30ae05..6dfc6dd 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -15,7 +15,7 @@ from pyexcel_io.plugin_api.abstract_sheet import ISheet class QuerysetsReader(ISheet): - """ turn querysets into an array """ + """turn querysets into an array""" def __init__(self, query_sets, column_names): self.name = DEFAULT_SHEET_NAME @@ -56,7 +56,7 @@ class QuerysetsReader(ISheet): def get_complex_attribute(row, attribute): - """ recursively get an attribute """ + """recursively get an attribute""" attributes = attribute.split("__") value = row try: @@ -68,7 +68,7 @@ def get_complex_attribute(row, attribute): def get_simple_attribute(row, attribute): - """ get dotted attribute """ + """get dotted attribute""" value = getattr(row, attribute) if isinstance(value, (datetime.date, datetime.time)): value = value.isoformat() diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 7e6625a..2a6b12c 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -32,7 +32,7 @@ class IOPluginInfo(PluginInfo): class IOPluginInfoChain(PluginInfoChain): - """provide custom functions to add a reader and a writer """ + """provide custom functions to add a reader and a writer""" def add_a_reader( self, @@ -40,7 +40,7 @@ class IOPluginInfoChain(PluginInfoChain): file_types=None, stream_type=None, ): - """ add pyexcle-io reader plugin info """ + """add pyexcle-io reader plugin info""" a_plugin_info = IOPluginInfo( READER_PLUGIN, self._get_abs_path(relative_plugin_class_path), @@ -55,7 +55,7 @@ class IOPluginInfoChain(PluginInfoChain): file_types=None, stream_type=None, ): - """ add pyexcle-io writer plugin info """ + """add pyexcle-io writer plugin info""" a_plugin_info = IOPluginInfo( WRITER_PLUGIN, self._get_abs_path(relative_plugin_class_path), @@ -66,7 +66,7 @@ class IOPluginInfoChain(PluginInfoChain): class IOPluginInfoChainV2(PluginInfoChain): - """provide custom functions to add a reader and a writer """ + """provide custom functions to add a reader and a writer""" def add_a_reader( self, @@ -75,7 +75,7 @@ class IOPluginInfoChainV2(PluginInfoChain): file_types=None, stream_type=None, ): - """ add pyexcle-io reader plugin info """ + """add pyexcle-io reader plugin info""" a_plugin_info = IOPluginInfo( READER_PLUGIN_V2, self._get_abs_path(relative_plugin_class_path), @@ -95,7 +95,7 @@ class IOPluginInfoChainV2(PluginInfoChain): file_types=(), stream_type=None, ): - """ add pyexcle-io writer plugin info """ + """add pyexcle-io writer plugin info""" a_plugin_info = IOPluginInfo( WRITER_PLUGIN_V2, self._get_abs_path(relative_plugin_class_path), @@ -124,7 +124,7 @@ class IOManager(PluginManager): _do_additional_registration(plugin_info) def register_a_plugin(self, cls, plugin_info): - """ for dynamically loaded plugin """ + """for dynamically loaded plugin""" PluginManager.register_a_plugin(self, cls, plugin_info) _do_additional_registration(plugin_info) @@ -158,7 +158,7 @@ class IOManager(PluginManager): ) def get_all_formats(self): - """ return all supported formats """ + """return all supported formats""" all_formats = set( list(self.registry.keys()) + list(self.known_plugins.keys()) ) @@ -174,7 +174,7 @@ class NewIOManager(IOManager): _do_additional_registration_for_new_plugins(plugin_info) def register_a_plugin(self, cls, plugin_info): - """ for dynamically loaded plugin """ + """for dynamically loaded plugin""" PluginManager.register_a_plugin(self, cls, plugin_info) _do_additional_registration_for_new_plugins(plugin_info) @@ -206,7 +206,7 @@ class NewIOManager(IOManager): ) def get_all_formats(self): - """ return all supported formats """ + """return all supported formats""" all_formats = set( [x.split("-")[1] for x in self.registry.keys()] + list(self.known_plugins.keys()) diff --git a/pyexcel_io/readers/csv_sheet.py b/pyexcel_io/readers/csv_sheet.py index 8ab6608..bac6f95 100644 --- a/pyexcel_io/readers/csv_sheet.py +++ b/pyexcel_io/readers/csv_sheet.py @@ -93,7 +93,7 @@ class CSVMemoryMapIterator(object): class CSVSheetReader(ISheet): - """ generic csv file reader""" + """generic csv file reader""" def __init__( self, @@ -121,7 +121,7 @@ class CSVSheetReader(ISheet): self._keywords = keywords def get_file_handle(self): - """ return me unicde reader for csv """ + """return me unicde reader for csv""" raise NotImplementedError("Please implement get_file_handle()") def row_iterator(self): @@ -166,7 +166,7 @@ class CSVSheetReader(ISheet): class CSVFileReader(CSVSheetReader): - """ read csv from phyical file """ + """read csv from phyical file""" def get_file_handle(self): unicode_reader = open( @@ -176,7 +176,7 @@ class CSVFileReader(CSVSheetReader): class CSVinMemoryReader(CSVSheetReader): - """ read csv file from memory """ + """read csv file from memory""" def get_file_handle(self): if isinstance(self._native_sheet.payload, compact.BytesIO): diff --git a/pyexcel_io/utils.py b/pyexcel_io/utils.py index 65477a0..f9f7950 100644 --- a/pyexcel_io/utils.py +++ b/pyexcel_io/utils.py @@ -69,10 +69,10 @@ def is_empty_array(array): def swap_empty_string_for_none(array): - """ replace empty string fields with None """ + """replace empty string fields with None""" def swap(value): - """ change empty string to None """ + """change empty string to None""" if value == "": return None diff --git a/pyexcel_io/writers/csv_sheet.py b/pyexcel_io/writers/csv_sheet.py index 3c2185f..f82a67c 100644 --- a/pyexcel_io/writers/csv_sheet.py +++ b/pyexcel_io/writers/csv_sheet.py @@ -75,7 +75,7 @@ class CSVFileWriter(ISheetWriter): class CSVMemoryWriter(CSVFileWriter): - """ Write csv to a memory stream """ + """Write csv to a memory stream""" def get_writer(self): self.file_handle = self._native_book diff --git a/pyexcel_io/writers/csvz_sheet.py b/pyexcel_io/writers/csvz_sheet.py index 26685cd..05fd4fd 100644 --- a/pyexcel_io/writers/csvz_sheet.py +++ b/pyexcel_io/writers/csvz_sheet.py @@ -14,7 +14,7 @@ from pyexcel_io.writers.csv_sheet import CSVFileWriter class CSVZipSheetWriter(CSVFileWriter): - """ handle the zipfile interface """ + """handle the zipfile interface""" def __init__(self, zipfile, sheetname, file_extension, **keywords): self.file_extension = file_extension From 842c3a0b402fd11ac5e6b92010872d38756cec28 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 8 Oct 2021 08:18:36 +0100 Subject: [PATCH 130/143] :books: ready for 0.6.5 release --- changelog.yml | 6 ++++++ pyexcel-io.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/changelog.yml b/changelog.yml index 2716ee5..6a2482d 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - "`#109`: enable ods3 to have datetime" + version: 0.6.5 + date: 08.10.2021 - changes: - action: updated details: diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 06bc073..24df243 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,7 +2,7 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.4 +version: 0.6.5 current_version: 0.6.5 release: 0.6.5 copyright_year: 2015-2020 From 5275c1c077e977f4259528a353cf28ac2ba128e6 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 8 Oct 2021 07:19:08 +0000 Subject: [PATCH 131/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- CHANGELOG.rst | 8 ++++++++ docs/source/conf.py | 2 +- docs/source/index.rst | 1 - 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9493eb4..8f13c04 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.6.5 - 08.10.2021 +-------------------------------------------------------------------------------- + +**updated** + +#. `#109 `_: enable ods3 to + have datetime + 0.6.4 - 31.10.2020 -------------------------------------------------------------------------------- diff --git a/docs/source/conf.py b/docs/source/conf.py index a830e40..f4d2ba0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ project = 'pyexcel-io' copyright = '2015-2020 Onni Software Ltd.' author = 'C.W.' # The short X.Y version -version = '0.6.4' +version = '0.6.5' # The full version, including alpha/beta/rc tags release = '0.6.5' diff --git a/docs/source/index.rst b/docs/source/index.rst index ac12734..baf6fd9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,6 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License -:Development: |release| :Released: |version| :Generated: |today| From 4ee4f9e742881b744b2474acf56c9d53c79ca1ae Mon Sep 17 00:00:00 2001 From: jaska Date: Fri, 29 Oct 2021 10:05:33 +0100 Subject: [PATCH 132/143] Update extensions.rst --- docs/source/extensions.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index d3d0ed3..d077e7c 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -26,7 +26,7 @@ we can use get_data() to read yaml file out. **Implement IReader** -First, let's impolement reader interface as below. Three implementations are required: +First, let's implement reader interface: 1. `content_array` attribute, is expected to be a list of `NamedContent` 2. `read_sheet` function, read sheet content by its index. @@ -63,6 +63,8 @@ files on physical disk. "memory" means a file stream. "content" means a string b :language: python :lines: 36-41 +Usually, this registration code was placed in __init__.py file at the top level of your +extension source tree. **Test your reader** From c42181f81bed2c57d4dc96ea253c5c910dd4773c Mon Sep 17 00:00:00 2001 From: jaska Date: Fri, 29 Oct 2021 22:06:50 +0100 Subject: [PATCH 133/143] Update extensions.rst --- docs/source/extensions.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index d077e7c..80c8b2f 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -41,10 +41,10 @@ First, let's implement reader interface: `YourSingleSheet` makes this simple task complex in order to show case its inner workings. Two abstract functions require implementation: -1. `row_iterator`: should return a row: either content arry or content index as long as - `column_iterator` understands +1. `row_iterator`: should return a row: either content arary or content index as long as + `column_iterator` can use it to return the cell value. -2. `column_iterator`: should return cell values one by one. +2. `column_iterator`: should iterate cell value from the given row. .. literalinclude:: ../../examples/custom_yaml_reader.py :language: python @@ -64,7 +64,7 @@ files on physical disk. "memory" means a file stream. "content" means a string b :lines: 36-41 Usually, this registration code was placed in __init__.py file at the top level of your -extension source tree. +extension source tree. You can take a look at any pyexcel plugins for reference. **Test your reader** From c49c2279da101d1e303193e4b787d9b81bc6080e Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 29 Oct 2021 21:07:21 +0000 Subject: [PATCH 134/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- CONTRIBUTORS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 5a6c33d..45bb32d 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -9,5 +9,5 @@ In alphabetical order: * `John Vandenberg `_ * `Stephen J. Fuhry `_ * `Stephen Rauch `_ -* `vinraspa `_ +* `Vincent Raspal `_ * `Víctor Antonio Hernández Monroy `_ From b66ccfc062b756e4068db484d21da6d9317c49b5 Mon Sep 17 00:00:00 2001 From: jaska Date: Fri, 29 Oct 2021 22:08:08 +0100 Subject: [PATCH 135/143] Update extensions.rst --- docs/source/extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/extensions.rst b/docs/source/extensions.rst index 80c8b2f..f436227 100644 --- a/docs/source/extensions.rst +++ b/docs/source/extensions.rst @@ -1,4 +1,4 @@ -Extend pyexcel-io Tutorial +Extend pyexcel-io for other excel or tabular formats ================================================================================ You are welcome to extend pyexcel-io to read and write more tabular formats. From d2cbb5decf6a1035b4157ed1156e97d9f7429e00 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 28 Jan 2022 23:47:23 +0000 Subject: [PATCH 136/143] :green_heart: convert print to log. resolves #112 --- changelog.yml | 6 ++++++ pyexcel-io.yml | 6 +++--- pyexcel_io/database/importers/django.py | 2 +- pyexcel_io/database/importers/sqlalchemy.py | 10 +++++++--- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/changelog.yml b/changelog.yml index 6a2482d..1a42dff 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - "`#112`: Log "Empty Row Warning" instead 'print' " + version: 0.6.6 + date: 31.1.2022 - changes: - action: updated details: diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 24df243..7c32477 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -3,9 +3,9 @@ project: "pyexcel-io" name: pyexcel-io nick_name: io version: 0.6.5 -current_version: 0.6.5 -release: 0.6.5 -copyright_year: 2015-2020 +current_version: 0.6.6 +release: 0.6.6 +copyright_year: 2015-2022 moban_command: false is_on_conda: true dependencies: diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 1c30f3d..101c43c 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -31,7 +31,7 @@ class DjangoModelWriter(ISheetWriter): def write_row(self, array): if is_empty_array(array): - print(constants.MESSAGE_EMPTY_ARRAY) + log.warning(constants.MESSAGE_EMPTY_ARRAY) else: new_array = swap_empty_string_for_none(array) if self.mapdict: diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 2cb0c30..859077f 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -7,10 +7,14 @@ :copyright: (c) 2014-2020 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ +import logging + import pyexcel_io.constants as constants from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none from pyexcel_io.plugin_api import IWriter, ISheetWriter +LOG = logging.getLogger(__name__) + class PyexcelSQLSkipRowException(Exception): """ @@ -35,14 +39,14 @@ class SQLTableWriter(ISheetWriter): def write_row(self, array): if is_empty_array(array): - print(constants.MESSAGE_EMPTY_ARRAY) + LOG.warning(constants.MESSAGE_EMPTY_ARRAY) else: new_array = swap_empty_string_for_none(array) try: self._write_row(new_array) except PyexcelSQLSkipRowException: - print(constants.MESSAGE_IGNORE_ROW) - print(new_array) + LOG.info(constants.MESSAGE_IGNORE_ROW) + LOG.info(new_array) def _write_row(self, array): new_array = array From 8c5f1e0de94f2ff1a675892e7fe3c1b857e6c78d Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 28 Jan 2022 23:48:06 +0000 Subject: [PATCH 137/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- LICENSE | 2 +- docs/source/conf.py | 4 ++-- docs/source/index.rst | 1 + setup.py | 8 ++++---- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/LICENSE b/LICENSE index 747c8b4..8bb697c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2020 by Onni Software Ltd. and its contributors +Copyright (c) 2015-2022 by Onni Software Ltd. and its contributors All rights reserved. Redistribution and use in source and binary forms of the software as well diff --git a/docs/source/conf.py b/docs/source/conf.py index f4d2ba0..51c3e67 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,12 +23,12 @@ DESCRIPTION = ( # -- Project information ----------------------------------------------------- project = 'pyexcel-io' -copyright = '2015-2020 Onni Software Ltd.' +copyright = '2015-2022 Onni Software Ltd.' author = 'C.W.' # The short X.Y version version = '0.6.5' # The full version, including alpha/beta/rc tags -release = '0.6.5' +release = '0.6.6' # -- General configuration --------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index baf6fd9..ac12734 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License +:Development: |release| :Released: |version| :Generated: |today| diff --git a/setup.py b/setup.py index ba6cd9e..90e27da 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "C.W." -VERSION = "0.6.5" +VERSION = "0.6.6" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( @@ -40,7 +40,7 @@ DESCRIPTION = ( "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.6.5.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.6.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -87,8 +87,8 @@ EXTRAS_REQUIRE = { PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) HERE = os.path.abspath(os.path.dirname(__file__)) -GS_COMMAND = ("gease pyexcel-io v0.6.5 " + - "Find 0.6.5 in changelog for more details") +GS_COMMAND = ("gease pyexcel-io v0.6.6 " + + "Find 0.6.6 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( From c7039124b2ab2d069f9955e38d10ac790c3c5389 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 28 Jan 2022 23:49:26 +0000 Subject: [PATCH 138/143] :lipstick: update copyright year --- pyexcel_io/__init__.py | 2 +- pyexcel_io/_compact.py | 2 +- pyexcel_io/book.py | 2 +- pyexcel_io/constants.py | 2 +- pyexcel_io/database/__init__.py | 2 +- pyexcel_io/database/common.py | 2 +- pyexcel_io/database/exporters/django.py | 2 +- pyexcel_io/database/exporters/sqlalchemy.py | 2 +- pyexcel_io/database/importers/django.py | 2 +- pyexcel_io/database/importers/sqlalchemy.py | 2 +- pyexcel_io/database/querysets.py | 2 +- pyexcel_io/exceptions.py | 2 +- pyexcel_io/io.py | 2 +- pyexcel_io/manager.py | 2 +- pyexcel_io/plugins.py | 2 +- pyexcel_io/readers/__init__.py | 2 +- pyexcel_io/readers/csv_sheet.py | 2 +- pyexcel_io/readers/csvz.py | 2 +- pyexcel_io/service.py | 2 +- pyexcel_io/sheet.py | 2 +- pyexcel_io/utils.py | 2 +- pyexcel_io/writers/__init__.py | 2 +- pyexcel_io/writers/csv_sheet.py | 2 +- pyexcel_io/writers/csvz_sheet.py | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pyexcel_io/__init__.py b/pyexcel_io/__init__.py index 0e836b2..f215e48 100644 --- a/pyexcel_io/__init__.py +++ b/pyexcel_io/__init__.py @@ -4,7 +4,7 @@ Uniform interface for reading/writing different excel file formats - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import logging diff --git a/pyexcel_io/_compact.py b/pyexcel_io/_compact.py index dd75146..e9827ba 100644 --- a/pyexcel_io/_compact.py +++ b/pyexcel_io/_compact.py @@ -4,7 +4,7 @@ Compatibles - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import sys diff --git a/pyexcel_io/book.py b/pyexcel_io/book.py index 7f0baa1..1948270 100644 --- a/pyexcel_io/book.py +++ b/pyexcel_io/book.py @@ -4,7 +4,7 @@ The io interface to file extensions - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import warnings diff --git a/pyexcel_io/constants.py b/pyexcel_io/constants.py index aa78a30..25e2823 100644 --- a/pyexcel_io/constants.py +++ b/pyexcel_io/constants.py @@ -4,7 +4,7 @@ Constants appeared in pyexcel - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License """ # flake8: noqa diff --git a/pyexcel_io/database/__init__.py b/pyexcel_io/database/__init__.py index 98c7df4..b7b4c6d 100644 --- a/pyexcel_io/database/__init__.py +++ b/pyexcel_io/database/__init__.py @@ -4,7 +4,7 @@ database data importer and exporter - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import IOPluginInfoChainV2 diff --git a/pyexcel_io/database/common.py b/pyexcel_io/database/common.py index 1008c1b..dd0f98d 100644 --- a/pyexcel_io/database/common.py +++ b/pyexcel_io/database/common.py @@ -4,7 +4,7 @@ Common classes shared among database importers and exporters - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index 018481a..1953ace 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -4,7 +4,7 @@ The lower level handler for django import and export - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugin_api import IReader diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index dbdc4c7..a1db305 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -4,7 +4,7 @@ The lower level handler for database import and export - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugin_api import IReader diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 101c43c..42f464f 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -4,7 +4,7 @@ The lower level handler for django import and export - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import logging diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 859077f..db10dc5 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -4,7 +4,7 @@ The lower level handler for database import and export - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import logging diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index 6dfc6dd..3ce08da 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -4,7 +4,7 @@ The lower level handler for querysets - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import datetime diff --git a/pyexcel_io/exceptions.py b/pyexcel_io/exceptions.py index b6d00aa..2c8ed9f 100644 --- a/pyexcel_io/exceptions.py +++ b/pyexcel_io/exceptions.py @@ -4,7 +4,7 @@ all possible exceptions - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 1606f83..8111959 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -4,7 +4,7 @@ The io interface to file extensions - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import os diff --git a/pyexcel_io/manager.py b/pyexcel_io/manager.py index 452ef18..9e15090 100644 --- a/pyexcel_io/manager.py +++ b/pyexcel_io/manager.py @@ -4,7 +4,7 @@ Control file streams - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io._compact import BytesIO, StringIO diff --git a/pyexcel_io/plugins.py b/pyexcel_io/plugins.py index 2a6b12c..495283a 100644 --- a/pyexcel_io/plugins.py +++ b/pyexcel_io/plugins.py @@ -4,7 +4,7 @@ factory for getting readers and writers - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.utils as ioutils diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py index e06ec19..c597292 100644 --- a/pyexcel_io/readers/__init__.py +++ b/pyexcel_io/readers/__init__.py @@ -4,7 +4,7 @@ file readers - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import IOPluginInfoChainV2 diff --git a/pyexcel_io/readers/csv_sheet.py b/pyexcel_io/readers/csv_sheet.py index bac6f95..fd6f76c 100644 --- a/pyexcel_io/readers/csv_sheet.py +++ b/pyexcel_io/readers/csv_sheet.py @@ -4,7 +4,7 @@ csv file reader - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import csv diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index 4556015..cae0383 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -4,7 +4,7 @@ The lower level csvz file format handler. - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import zipfile diff --git a/pyexcel_io/service.py b/pyexcel_io/service.py index 02d0a0e..958ee1e 100644 --- a/pyexcel_io/service.py +++ b/pyexcel_io/service.py @@ -4,7 +4,7 @@ provide service code to downstream projects - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import re diff --git a/pyexcel_io/sheet.py b/pyexcel_io/sheet.py index b7c7015..da730e3 100644 --- a/pyexcel_io/sheet.py +++ b/pyexcel_io/sheet.py @@ -4,7 +4,7 @@ The io interface to file extensions - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants diff --git a/pyexcel_io/utils.py b/pyexcel_io/utils.py index f9f7950..f6f890d 100644 --- a/pyexcel_io/utils.py +++ b/pyexcel_io/utils.py @@ -4,7 +4,7 @@ utility functions - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import pyexcel_io.constants as constants diff --git a/pyexcel_io/writers/__init__.py b/pyexcel_io/writers/__init__.py index 5a8ce4b..2beeaf7 100644 --- a/pyexcel_io/writers/__init__.py +++ b/pyexcel_io/writers/__init__.py @@ -4,7 +4,7 @@ file writers - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import IOPluginInfoChainV2 diff --git a/pyexcel_io/writers/csv_sheet.py b/pyexcel_io/writers/csv_sheet.py index f82a67c..422dbed 100644 --- a/pyexcel_io/writers/csv_sheet.py +++ b/pyexcel_io/writers/csv_sheet.py @@ -4,7 +4,7 @@ The lower level csv file format writer - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import csv diff --git a/pyexcel_io/writers/csvz_sheet.py b/pyexcel_io/writers/csvz_sheet.py index 05fd4fd..fe0cd22 100644 --- a/pyexcel_io/writers/csvz_sheet.py +++ b/pyexcel_io/writers/csvz_sheet.py @@ -4,7 +4,7 @@ The lower level csvz file format handler. - :copyright: (c) 2014-2020 by Onni Software Ltd. + :copyright: (c) 2014-2022 by Onni Software Ltd. :license: New BSD License, see LICENSE for more details """ import csv From a17e12c672f5d353e60ccf907855003d915395c1 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 28 Jan 2022 23:51:06 +0000 Subject: [PATCH 139/143] :bug: fix changelog --- changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.yml b/changelog.yml index 1a42dff..e9244a9 100644 --- a/changelog.yml +++ b/changelog.yml @@ -4,7 +4,7 @@ releases: - changes: - action: updated details: - - "`#112`: Log "Empty Row Warning" instead 'print' " + - "`#112`: Log Empty Row Warning instead 'print' " version: 0.6.6 date: 31.1.2022 - changes: From 1d779d3481957144a3ba34a21d1027fbe7cedfae Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 28 Jan 2022 23:52:01 +0000 Subject: [PATCH 140/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8f13c04..5cd5f17 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.6.6 - 31.1.2022 +-------------------------------------------------------------------------------- + +**updated** + +#. `#112 `_: Log Empty Row + Warning instead 'print' + 0.6.5 - 08.10.2021 -------------------------------------------------------------------------------- From eebf47310c09e630c583e71349d3557933d1ec59 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 29 Jan 2022 17:39:50 +0000 Subject: [PATCH 141/143] :books: update workflow names --- .github/workflows/pythonpackage.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a7a148a..e19f742 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -1,4 +1,4 @@ -name: Python package +name: Unit tests on ubuntu on: [push] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 810f878..801d2cd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: run_tests +name: Run unit tests on Windows and Mac on: [push, pull_request] From 5093b97ad0b2364d642514c9d9ee3e099835c0ed Mon Sep 17 00:00:00 2001 From: jaska Date: Sat, 29 Jan 2022 19:45:47 +0000 Subject: [PATCH 142/143] Update pyexcel-io.yml --- pyexcel-io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 7c32477..7dad5e1 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -2,7 +2,7 @@ overrides: "pyexcel.yaml" project: "pyexcel-io" name: pyexcel-io nick_name: io -version: 0.6.5 +version: 0.6.6 current_version: 0.6.6 release: 0.6.6 copyright_year: 2015-2022 From 1caf894d0a3fda2e7392984690aead654c17464c Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 29 Jan 2022 19:46:13 +0000 Subject: [PATCH 143/143] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- docs/source/conf.py | 2 +- docs/source/index.rst | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 51c3e67..7d8e0cc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ project = 'pyexcel-io' copyright = '2015-2022 Onni Software Ltd.' author = 'C.W.' # The short X.Y version -version = '0.6.5' +version = '0.6.6' # The full version, including alpha/beta/rc tags release = '0.6.6' diff --git a/docs/source/index.rst b/docs/source/index.rst index ac12734..baf6fd9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,6 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License -:Development: |release| :Released: |version| :Generated: |today|