From cd081dddaa7b7dce6c4338d705781fbe3e7d3c3d Mon Sep 17 00:00:00 2001 From: jaska Date: Thu, 29 Oct 2020 07:18:43 +0000 Subject: [PATCH] Django jump column (#103) * :sparkles: support skipping columns when importing a spreadsheet. fix #102 * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :books: update typo in changelog.yml * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst Co-authored-by: chfw --- CHANGELOG.rst | 8 ++++++++ README.rst | 2 +- changelog.yml | 6 ++++++ docs/source/conf.py | 2 +- docs/source/index.rst | 1 + pyexcel-io.yml | 4 ++-- pyexcel_io/database/common.py | 14 +++++++++----- pyexcel_io/database/importers/django.py | 9 ++++++++- pyexcel_io/database/importers/sqlalchemy.py | 15 +++++++++------ setup.py | 8 ++++---- tests/test_django_book.py | 18 +++++++++++++++--- 11 files changed, 64 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 92e656b..5289adb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.6.4 - tbd +-------------------------------------------------------------------------------- + +**updated** + +#. `#102 `_: skip columns from + imported excel sheet. + 0.6.3 - 12.10.2020 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 4ec5d4e..4dabd67 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ pyexcel-io - Let you focus on data, instead of file formats :target: https://anaconda.org/conda-forge/pyexcel-io .. image:: https://pepy.tech/badge/pyexcel-io/month - :target: https://pepy.tech/project/pyexcel-io/month + :target: https://pepy.tech/project/pyexcel-io .. image:: https://anaconda.org/conda-forge/pyexcel-io/badges/downloads.svg :target: https://anaconda.org/conda-forge/pyexcel-io diff --git a/changelog.yml b/changelog.yml index 47f4053..7c84efa 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,12 @@ name: pyexcel-io organisation: pyexcel releases: +- changes: + - action: updated + details: + - "`#102`: skip columns from imported excel sheet." + version: 0.6.4 + date: tbd - changes: - action: fixed details: diff --git a/docs/source/conf.py b/docs/source/conf.py index 4595678..115126c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ author = 'C.W.' # The short X.Y version version = '0.6.3' # The full version, including alpha/beta/rc tags -release = '0.6.3' +release = '0.6.4' # -- General configuration --------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 00332e4..ad3a030 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,7 @@ :Source code: http://github.com/pyexcel/pyexcel-io.git :Issues: http://github.com/pyexcel/pyexcel-io/issues :License: New BSD License +:Development: |release| :Released: |version| :Generated: |today| diff --git a/pyexcel-io.yml b/pyexcel-io.yml index 8b43aff..a4a3683 100644 --- a/pyexcel-io.yml +++ b/pyexcel-io.yml @@ -3,8 +3,8 @@ project: "pyexcel-io" name: pyexcel-io nick_name: io version: 0.6.3 -current_version: 0.6.3 -release: 0.6.3 +current_version: 0.6.4 +release: 0.6.4 copyright_year: 2015-2020 moban_command: false is_on_conda: true diff --git a/pyexcel_io/database/common.py b/pyexcel_io/database/common.py index 1f60ebd..a4a1994 100644 --- a/pyexcel_io/database/common.py +++ b/pyexcel_io/database/common.py @@ -86,11 +86,15 @@ class DjangoModelImportAdapter(DjangoModelExportAdapter): self.__column_name_mapping_dict.output = None elif isinstance(self.__column_name_mapping_dict.input, dict): if self.__column_names.input: - self.__column_names.output = [ - self.__column_name_mapping_dict.input[name] - for name in self.__column_names.input - ] - self.__column_name_mapping_dict.output = None + self.__column_names.output = [] + indices = [] + for index, name in enumerate(self.__column_names.input): + if name in self.__column_name_mapping_dict.input: + self.__column_names.output.append( + self.__column_name_mapping_dict.input[name] + ) + indices.append(index) + self.__column_name_mapping_dict.output = indices if self.__column_names.output is None: self.__column_names.output = self.__column_names.input diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py index 9d7307e..78592cf 100644 --- a/pyexcel_io/database/importers/django.py +++ b/pyexcel_io/database/importers/django.py @@ -33,13 +33,20 @@ class DjangoModelWriter(ISheetWriter): print(constants.MESSAGE_EMPTY_ARRAY) else: new_array = swap_empty_string_for_none(array) + if self.__mapdict: + another_new_array = [] + for index, element in enumerate(new_array): + if index in self.__mapdict: + another_new_array.append(element) + new_array = another_new_array model_to_be_created = new_array if self.__initializer is not None: model_to_be_created = self.__initializer(new_array) if model_to_be_created: + row = dict(zip(self.__column_names, model_to_be_created)) self.__objs.append( self.__model( - **dict(zip(self.__column_names, model_to_be_created)) + **row ) ) diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py index 5c5cd27..cf2705a 100644 --- a/pyexcel_io/database/importers/sqlalchemy.py +++ b/pyexcel_io/database/importers/sqlalchemy.py @@ -45,7 +45,14 @@ class SQLTableWriter(ISheetWriter): print(new_array) def _write_row(self, array): - row = dict(zip(self.adapter.column_names, array)) + new_array = array + if self.adapter.column_name_mapping_dict: + another_new_array = [] + for index, element in enumerate(new_array): + if index in self.adapter.column_name_mapping_dict: + another_new_array.append(element) + new_array = another_new_array + row = dict(zip(self.adapter.column_names, new_array)) obj = None if self.adapter.row_initializer: # allow initinalizer to return None @@ -54,11 +61,7 @@ class SQLTableWriter(ISheetWriter): if obj is None: obj = self.adapter.table() for name in self.adapter.column_names: - if self.adapter.column_name_mapping_dict is not None: - key = self.adapter.column_name_mapping_dict[name] - else: - key = name - setattr(obj, key, row[name]) + setattr(obj, name, row[name]) self.importer.session.add(obj) if self.__auto_commit and self.__bulk_size != float("inf"): self.__count += 1 diff --git a/setup.py b/setup.py index 967cc22..267d6a2 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ except (ValueError, UnicodeError, locale.Error): NAME = "pyexcel-io" AUTHOR = "C.W." -VERSION = "0.6.3" +VERSION = "0.6.4" EMAIL = "info@pyexcel.org" LICENSE = "New BSD" DESCRIPTION = ( @@ -40,7 +40,7 @@ DESCRIPTION = ( "format and to/from databases" ) URL = "https://github.com/pyexcel/pyexcel-io" -DOWNLOAD_URL = "%s/archive/0.6.3.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.4.tar.gz" % URL FILES = ["README.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -87,8 +87,8 @@ EXTRAS_REQUIRE = { PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) HERE = os.path.abspath(os.path.dirname(__file__)) -GS_COMMAND = ("gease pyexcel-io v0.6.3 " + - "Find 0.6.3 in changelog for more details") +GS_COMMAND = ("gease pyexcel-io v0.6.4 " + + "Find 0.6.4 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( diff --git a/tests/test_django_book.py b/tests/test_django_book.py index 5db3e7c..a95d7ab 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -158,10 +158,10 @@ class TestSheet: writer = DjangoModelWriter(None, adapter) writer.write_array(self.data[1:]) writer.close() - assert model.objects.objs == [ + eq_(model.objects.objs, [ {"Y": 2, "X": 2, "Z": 3}, {"Y": 5, "X": 5, "Z": 6}, - ] + ]) def test_sheet_save_to_django_model_skip_me(self): model = FakeDjangoModel() @@ -178,7 +178,7 @@ class TestSheet: writer = DjangoModelWriter(None, adapter) writer.write_array(self.data[1:]) writer.close() - assert model.objects.objs == [{"Y": 2, "X": 1, "Z": 3}] + eq_(model.objects.objs, [{"Y": 2, "X": 1, "Z": 3}]) def test_load_sheet_from_django_model(self): model = FakeDjangoModel() @@ -241,6 +241,18 @@ class TestSheet: writer.close() eq_(model.objects.objs, self.result) + def test_jumping_columns(self): + data2 = [["D", "A", "B", "C"], [1, 1, 2, 3], [10, 4, 5, 6]] + mapdict = {"C": "Z", "A": "X", "B": "Y"} + model = FakeDjangoModel() + adapter = DjangoModelImportAdapter(model) + adapter.column_names = data2[0] + adapter.column_name_mapping_dict = mapdict + writer = DjangoModelWriter(None, adapter) + writer.write_array(data2[1:]) + writer.close() + eq_(model.objects.objs, self.result) + def test_empty_model(self): model = FakeDjangoModel() reader = DjangoModelReader(model)