Merge branch 'wip/github'
This commit is contained in:
commit
81bbd9e685
|
@ -0,0 +1,4 @@
|
||||||
|
# These are supported funding model platforms
|
||||||
|
|
||||||
|
github: chfw
|
||||||
|
patreon: chfw
|
|
@ -0,0 +1,8 @@
|
||||||
|
With your PR, here is a check list:
|
||||||
|
|
||||||
|
- [ ] Has test cases written?
|
||||||
|
- [ ] Has all code lines tested?
|
||||||
|
- [ ] Has `make format` been run?
|
||||||
|
- [ ] Please update CHANGELOG.yml(not CHANGELOG.rst)
|
||||||
|
- [ ] Has fair amount of documentation if your change is complex
|
||||||
|
- [ ] Agree on NEW BSD License for your contribution
|
|
@ -0,0 +1,20 @@
|
||||||
|
name: lint
|
||||||
|
|
||||||
|
on: [push, pull_request]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
name: lint code
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v1
|
||||||
|
with:
|
||||||
|
python-version: 3.8
|
||||||
|
- name: lint
|
||||||
|
run: |
|
||||||
|
pip --use-deprecated=legacy-resolver install flake8
|
||||||
|
pip --use-deprecated=legacy-resolver install -r tests/requirements.txt
|
||||||
|
flake8 --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long .
|
||||||
|
python setup.py checkdocs
|
|
@ -0,0 +1,28 @@
|
||||||
|
on: [push]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
run_moban:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
name: synchronize templates via moban
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
ref: ${{ github.head_ref }}
|
||||||
|
token: ${{ secrets.PAT }}
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v1
|
||||||
|
with:
|
||||||
|
python-version: '3.7'
|
||||||
|
- name: check changes
|
||||||
|
run: |
|
||||||
|
pip install moban gitfs2 pypifs moban-jinja2-github moban-ansible
|
||||||
|
moban
|
||||||
|
git status
|
||||||
|
git diff --exit-code
|
||||||
|
- name: Auto-commit
|
||||||
|
if: failure()
|
||||||
|
uses: stefanzweifel/git-auto-commit-action@v4
|
||||||
|
with:
|
||||||
|
commit_message: >-
|
||||||
|
This is an auto-commit, updating project meta data,
|
||||||
|
such as changelog.rst, contributors.rst
|
|
@ -0,0 +1,30 @@
|
||||||
|
name: Unit tests on ubuntu
|
||||||
|
|
||||||
|
on: [push]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
max-parallel: 4
|
||||||
|
matrix:
|
||||||
|
python-version: [3.6, 3.7, 3.8]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v1
|
||||||
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
|
uses: actions/setup-python@v1
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
- name: Install dependencies
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
pip install -r requirements.txt
|
||||||
|
- name: Lint with flake8
|
||||||
|
run: |
|
||||||
|
make install_test format git-diff-check lint
|
||||||
|
- name: Test
|
||||||
|
run: |
|
||||||
|
pip install -r tests/requirements.txt
|
||||||
|
make
|
|
@ -0,0 +1,26 @@
|
||||||
|
name: Upload Python Package
|
||||||
|
|
||||||
|
on:
|
||||||
|
release:
|
||||||
|
types: [created]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v1
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v1
|
||||||
|
with:
|
||||||
|
python-version: '3.x'
|
||||||
|
- name: Install dependencies
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
pip install setuptools wheel twine
|
||||||
|
- name: Build and publish
|
||||||
|
env:
|
||||||
|
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
|
||||||
|
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
|
||||||
|
run: |
|
||||||
|
python setup.py sdist bdist_wheel
|
||||||
|
twine upload dist/*
|
|
@ -0,0 +1,35 @@
|
||||||
|
name: Run unit tests on Windows and Mac
|
||||||
|
|
||||||
|
on: [push, pull_request]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
python-version: [3.6, 3.7, 3.8, 3.9]
|
||||||
|
os: [macOs-latest, ubuntu-latest, windows-latest]
|
||||||
|
exclude:
|
||||||
|
- os: macOs-latest
|
||||||
|
python-version: 3.6
|
||||||
|
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
name: run tests
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v1
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
- name: install
|
||||||
|
run: |
|
||||||
|
pip --use-deprecated=legacy-resolver install -r requirements.txt
|
||||||
|
pip --use-deprecated=legacy-resolver install -r tests/requirements.txt
|
||||||
|
- name: test
|
||||||
|
run: |
|
||||||
|
pip freeze
|
||||||
|
nosetests --verbosity=3 --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io
|
||||||
|
- name: Upload coverage
|
||||||
|
uses: codecov/codecov-action@v1
|
||||||
|
with:
|
||||||
|
name: ${{ matrix.os }} Python ${{ matrix.python-version }}
|
|
@ -1,8 +1,546 @@
|
||||||
*.pyc
|
# moban hashes
|
||||||
*~
|
.moban.hashes
|
||||||
|
|
||||||
|
# Extra rules from https://github.com/github/gitignore/
|
||||||
|
# Python rules
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
.coverage
|
.coverage
|
||||||
pyexcel*-info
|
.coverage.*
|
||||||
build
|
.cache
|
||||||
dist
|
nosetests.xml
|
||||||
tmp.db
|
coverage.xml
|
||||||
.idea/*
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
#Pipfile.lock
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# VirtualEnv rules
|
||||||
|
# Virtualenv
|
||||||
|
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
|
||||||
|
.Python
|
||||||
|
[Bb]in
|
||||||
|
[Ii]nclude
|
||||||
|
[Ll]ib
|
||||||
|
[Ll]ib64
|
||||||
|
[Ll]ocal
|
||||||
|
[Ss]cripts
|
||||||
|
pyvenv.cfg
|
||||||
|
.venv
|
||||||
|
pip-selfcheck.json
|
||||||
|
|
||||||
|
# Linux rules
|
||||||
|
*~
|
||||||
|
|
||||||
|
# temporary files which can be created if a process still has a handle open of a deleted file
|
||||||
|
.fuse_hidden*
|
||||||
|
|
||||||
|
# KDE directory preferences
|
||||||
|
.directory
|
||||||
|
|
||||||
|
# Linux trash folder which might appear on any partition or disk
|
||||||
|
.Trash-*
|
||||||
|
|
||||||
|
# .nfs files are created when an open file is removed but is still being accessed
|
||||||
|
.nfs*
|
||||||
|
|
||||||
|
# Windows rules
|
||||||
|
# Windows thumbnail cache files
|
||||||
|
Thumbs.db
|
||||||
|
Thumbs.db:encryptable
|
||||||
|
ehthumbs.db
|
||||||
|
ehthumbs_vista.db
|
||||||
|
|
||||||
|
# Dump file
|
||||||
|
*.stackdump
|
||||||
|
|
||||||
|
# Folder config file
|
||||||
|
[Dd]esktop.ini
|
||||||
|
|
||||||
|
# Recycle Bin used on file shares
|
||||||
|
$RECYCLE.BIN/
|
||||||
|
|
||||||
|
# Windows Installer files
|
||||||
|
*.cab
|
||||||
|
*.msi
|
||||||
|
*.msix
|
||||||
|
*.msm
|
||||||
|
*.msp
|
||||||
|
|
||||||
|
# Windows shortcuts
|
||||||
|
*.lnk
|
||||||
|
|
||||||
|
# macOS rules
|
||||||
|
# General
|
||||||
|
.DS_Store
|
||||||
|
.AppleDouble
|
||||||
|
.LSOverride
|
||||||
|
|
||||||
|
# Icon must end with two \r
|
||||||
|
Icon
|
||||||
|
|
||||||
|
|
||||||
|
# Thumbnails
|
||||||
|
._*
|
||||||
|
|
||||||
|
# Files that might appear in the root of a volume
|
||||||
|
.DocumentRevisions-V100
|
||||||
|
.fseventsd
|
||||||
|
.Spotlight-V100
|
||||||
|
.TemporaryItems
|
||||||
|
.Trashes
|
||||||
|
.VolumeIcon.icns
|
||||||
|
.com.apple.timemachine.donotpresent
|
||||||
|
|
||||||
|
# Directories potentially created on remote AFP share
|
||||||
|
.AppleDB
|
||||||
|
.AppleDesktop
|
||||||
|
Network Trash Folder
|
||||||
|
Temporary Items
|
||||||
|
.apdisk
|
||||||
|
|
||||||
|
# Emacs rules
|
||||||
|
# -*- mode: gitignore; -*-
|
||||||
|
*~
|
||||||
|
\#*\#
|
||||||
|
/.emacs.desktop
|
||||||
|
/.emacs.desktop.lock
|
||||||
|
*.elc
|
||||||
|
auto-save-list
|
||||||
|
tramp
|
||||||
|
.\#*
|
||||||
|
|
||||||
|
# Org-mode
|
||||||
|
.org-id-locations
|
||||||
|
*_archive
|
||||||
|
|
||||||
|
# flymake-mode
|
||||||
|
*_flymake.*
|
||||||
|
|
||||||
|
# eshell files
|
||||||
|
/eshell/history
|
||||||
|
/eshell/lastdir
|
||||||
|
|
||||||
|
# elpa packages
|
||||||
|
/elpa/
|
||||||
|
|
||||||
|
# reftex files
|
||||||
|
*.rel
|
||||||
|
|
||||||
|
# AUCTeX auto folder
|
||||||
|
/auto/
|
||||||
|
|
||||||
|
# cask packages
|
||||||
|
.cask/
|
||||||
|
dist/
|
||||||
|
|
||||||
|
# Flycheck
|
||||||
|
flycheck_*.el
|
||||||
|
|
||||||
|
# server auth directory
|
||||||
|
/server/
|
||||||
|
|
||||||
|
# projectiles files
|
||||||
|
.projectile
|
||||||
|
|
||||||
|
# directory configuration
|
||||||
|
.dir-locals.el
|
||||||
|
|
||||||
|
# network security
|
||||||
|
/network-security.data
|
||||||
|
|
||||||
|
|
||||||
|
# Vim rules
|
||||||
|
# Swap
|
||||||
|
[._]*.s[a-v][a-z]
|
||||||
|
!*.svg # comment out if you don't need vector files
|
||||||
|
[._]*.sw[a-p]
|
||||||
|
[._]s[a-rt-v][a-z]
|
||||||
|
[._]ss[a-gi-z]
|
||||||
|
[._]sw[a-p]
|
||||||
|
|
||||||
|
# Session
|
||||||
|
Session.vim
|
||||||
|
Sessionx.vim
|
||||||
|
|
||||||
|
# Temporary
|
||||||
|
.netrwhist
|
||||||
|
*~
|
||||||
|
# Auto-generated tag files
|
||||||
|
tags
|
||||||
|
# Persistent undo
|
||||||
|
[._]*.un~
|
||||||
|
|
||||||
|
# JetBrains rules
|
||||||
|
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||||
|
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||||
|
|
||||||
|
# User-specific stuff
|
||||||
|
.idea/**/workspace.xml
|
||||||
|
.idea/**/tasks.xml
|
||||||
|
.idea/**/usage.statistics.xml
|
||||||
|
.idea/**/dictionaries
|
||||||
|
.idea/**/shelf
|
||||||
|
|
||||||
|
# Generated files
|
||||||
|
.idea/**/contentModel.xml
|
||||||
|
|
||||||
|
# Sensitive or high-churn files
|
||||||
|
.idea/**/dataSources/
|
||||||
|
.idea/**/dataSources.ids
|
||||||
|
.idea/**/dataSources.local.xml
|
||||||
|
.idea/**/sqlDataSources.xml
|
||||||
|
.idea/**/dynamic.xml
|
||||||
|
.idea/**/uiDesigner.xml
|
||||||
|
.idea/**/dbnavigator.xml
|
||||||
|
|
||||||
|
# Gradle
|
||||||
|
.idea/**/gradle.xml
|
||||||
|
.idea/**/libraries
|
||||||
|
|
||||||
|
# Gradle and Maven with auto-import
|
||||||
|
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||||
|
# since they will be recreated, and may cause churn. Uncomment if using
|
||||||
|
# auto-import.
|
||||||
|
# .idea/artifacts
|
||||||
|
# .idea/compiler.xml
|
||||||
|
# .idea/jarRepositories.xml
|
||||||
|
# .idea/modules.xml
|
||||||
|
# .idea/*.iml
|
||||||
|
# .idea/modules
|
||||||
|
# *.iml
|
||||||
|
# *.ipr
|
||||||
|
|
||||||
|
# CMake
|
||||||
|
cmake-build-*/
|
||||||
|
|
||||||
|
# Mongo Explorer plugin
|
||||||
|
.idea/**/mongoSettings.xml
|
||||||
|
|
||||||
|
# File-based project format
|
||||||
|
*.iws
|
||||||
|
|
||||||
|
# IntelliJ
|
||||||
|
out/
|
||||||
|
|
||||||
|
# mpeltonen/sbt-idea plugin
|
||||||
|
.idea_modules/
|
||||||
|
|
||||||
|
# JIRA plugin
|
||||||
|
atlassian-ide-plugin.xml
|
||||||
|
|
||||||
|
# Cursive Clojure plugin
|
||||||
|
.idea/replstate.xml
|
||||||
|
|
||||||
|
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||||
|
com_crashlytics_export_strings.xml
|
||||||
|
crashlytics.properties
|
||||||
|
crashlytics-build.properties
|
||||||
|
fabric.properties
|
||||||
|
|
||||||
|
# Editor-based Rest Client
|
||||||
|
.idea/httpRequests
|
||||||
|
|
||||||
|
# Android studio 3.1+ serialized cache file
|
||||||
|
.idea/caches/build_file_checksums.ser
|
||||||
|
|
||||||
|
# SublimeText rules
|
||||||
|
# Cache files for Sublime Text
|
||||||
|
*.tmlanguage.cache
|
||||||
|
*.tmPreferences.cache
|
||||||
|
*.stTheme.cache
|
||||||
|
|
||||||
|
# Workspace files are user-specific
|
||||||
|
*.sublime-workspace
|
||||||
|
|
||||||
|
# Project files should be checked into the repository, unless a significant
|
||||||
|
# proportion of contributors will probably not be using Sublime Text
|
||||||
|
# *.sublime-project
|
||||||
|
|
||||||
|
# SFTP configuration file
|
||||||
|
sftp-config.json
|
||||||
|
sftp-config-alt*.json
|
||||||
|
|
||||||
|
# Package control specific files
|
||||||
|
Package Control.last-run
|
||||||
|
Package Control.ca-list
|
||||||
|
Package Control.ca-bundle
|
||||||
|
Package Control.system-ca-bundle
|
||||||
|
Package Control.cache/
|
||||||
|
Package Control.ca-certs/
|
||||||
|
Package Control.merged-ca-bundle
|
||||||
|
Package Control.user-ca-bundle
|
||||||
|
oscrypto-ca-bundle.crt
|
||||||
|
bh_unicode_properties.cache
|
||||||
|
|
||||||
|
# Sublime-github package stores a github token in this file
|
||||||
|
# https://packagecontrol.io/packages/sublime-github
|
||||||
|
GitHub.sublime-settings
|
||||||
|
|
||||||
|
# KDevelop4 rules
|
||||||
|
*.kdev4
|
||||||
|
.kdev4/
|
||||||
|
|
||||||
|
# Kate rules
|
||||||
|
# Swap Files #
|
||||||
|
.*.kate-swp
|
||||||
|
.swp.*
|
||||||
|
|
||||||
|
# TextMate rules
|
||||||
|
*.tmproj
|
||||||
|
*.tmproject
|
||||||
|
tmtags
|
||||||
|
|
||||||
|
# VisualStudioCode rules
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/settings.json
|
||||||
|
!.vscode/tasks.json
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/extensions.json
|
||||||
|
*.code-workspace
|
||||||
|
|
||||||
|
# Local History for Visual Studio Code
|
||||||
|
.history/
|
||||||
|
|
||||||
|
# Xcode rules
|
||||||
|
# Xcode
|
||||||
|
#
|
||||||
|
# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
|
||||||
|
|
||||||
|
## User settings
|
||||||
|
xcuserdata/
|
||||||
|
|
||||||
|
## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9)
|
||||||
|
*.xcscmblueprint
|
||||||
|
*.xccheckout
|
||||||
|
|
||||||
|
## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4)
|
||||||
|
build/
|
||||||
|
DerivedData/
|
||||||
|
*.moved-aside
|
||||||
|
*.pbxuser
|
||||||
|
!default.pbxuser
|
||||||
|
*.mode1v3
|
||||||
|
!default.mode1v3
|
||||||
|
*.mode2v3
|
||||||
|
!default.mode2v3
|
||||||
|
*.perspectivev3
|
||||||
|
!default.perspectivev3
|
||||||
|
|
||||||
|
## Gcc Patch
|
||||||
|
/*.gcno
|
||||||
|
|
||||||
|
# Eclipse rules
|
||||||
|
.metadata
|
||||||
|
bin/
|
||||||
|
tmp/
|
||||||
|
*.tmp
|
||||||
|
*.bak
|
||||||
|
*.swp
|
||||||
|
*~.nib
|
||||||
|
local.properties
|
||||||
|
.settings/
|
||||||
|
.loadpath
|
||||||
|
.recommenders
|
||||||
|
|
||||||
|
# External tool builders
|
||||||
|
.externalToolBuilders/
|
||||||
|
|
||||||
|
# Locally stored "Eclipse launch configurations"
|
||||||
|
*.launch
|
||||||
|
|
||||||
|
# PyDev specific (Python IDE for Eclipse)
|
||||||
|
*.pydevproject
|
||||||
|
|
||||||
|
# CDT-specific (C/C++ Development Tooling)
|
||||||
|
.cproject
|
||||||
|
|
||||||
|
# CDT- autotools
|
||||||
|
.autotools
|
||||||
|
|
||||||
|
# Java annotation processor (APT)
|
||||||
|
.factorypath
|
||||||
|
|
||||||
|
# PDT-specific (PHP Development Tools)
|
||||||
|
.buildpath
|
||||||
|
|
||||||
|
# sbteclipse plugin
|
||||||
|
.target
|
||||||
|
|
||||||
|
# Tern plugin
|
||||||
|
.tern-project
|
||||||
|
|
||||||
|
# TeXlipse plugin
|
||||||
|
.texlipse
|
||||||
|
|
||||||
|
# STS (Spring Tool Suite)
|
||||||
|
.springBeans
|
||||||
|
|
||||||
|
# Code Recommenders
|
||||||
|
.recommenders/
|
||||||
|
|
||||||
|
# Annotation Processing
|
||||||
|
.apt_generated/
|
||||||
|
.apt_generated_test/
|
||||||
|
|
||||||
|
# Scala IDE specific (Scala & Java development for Eclipse)
|
||||||
|
.cache-main
|
||||||
|
.scala_dependencies
|
||||||
|
.worksheet
|
||||||
|
|
||||||
|
# Uncomment this line if you wish to ignore the project description file.
|
||||||
|
# Typically, this file would be tracked if it contains build/dependency configurations:
|
||||||
|
#.project
|
||||||
|
|
||||||
|
# TortoiseGit rules
|
||||||
|
# Project-level settings
|
||||||
|
/.tgitconfig
|
||||||
|
|
||||||
|
# Tags rules
|
||||||
|
# Ignore tags created by etags, ctags, gtags (GNU global) and cscope
|
||||||
|
TAGS
|
||||||
|
.TAGS
|
||||||
|
!TAGS/
|
||||||
|
tags
|
||||||
|
.tags
|
||||||
|
!tags/
|
||||||
|
gtags.files
|
||||||
|
GTAGS
|
||||||
|
GRTAGS
|
||||||
|
GPATH
|
||||||
|
GSYMS
|
||||||
|
cscope.files
|
||||||
|
cscope.out
|
||||||
|
cscope.in.out
|
||||||
|
cscope.po.out
|
||||||
|
|
||||||
|
|
||||||
|
# remove moban hash dictionary
|
||||||
|
.moban.hashes
|
||||||
|
|
|
@ -0,0 +1,10 @@
|
||||||
|
[settings]
|
||||||
|
line_length=79
|
||||||
|
known_first_party=lml, pyexcel
|
||||||
|
known_third_party=nose
|
||||||
|
indent=' '
|
||||||
|
multi_line_output=3
|
||||||
|
length_sort=1
|
||||||
|
default_section=FIRSTPARTY
|
||||||
|
no_lines_before=LOCALFOLDER
|
||||||
|
sections=FUTURE,STDLIB,FIRSTPARTY,THIRDPARTY,LOCALFOLDER
|
|
@ -0,0 +1,13 @@
|
||||||
|
{% extends "travis.yml.jj2" %}
|
||||||
|
{%block extra_matrix %}
|
||||||
|
env:
|
||||||
|
- MINREQ=1
|
||||||
|
{%endblock%}
|
||||||
|
{%block custom_python_versions%}
|
||||||
|
python:
|
||||||
|
- 3.8
|
||||||
|
- 3.7
|
||||||
|
- 3.6
|
||||||
|
{%endblock%}
|
||||||
|
{%block pypi_deployment%}
|
||||||
|
{%endblock %}
|
|
@ -12,4 +12,3 @@ def setup(app):
|
||||||
|
|
||||||
|
|
||||||
{%endblock%}
|
{%endblock%}
|
||||||
|
|
|
@ -57,7 +57,9 @@ get_data(.., library='pyexcel-ods')
|
||||||
============= ======= ======== ======= ======== ======== ========
|
============= ======= ======== ======= ======== ======== ========
|
||||||
`pyexcel-io`_ `xls`_ `xlsx`_ `ods`_ `ods3`_ `odsr`_ `xlsxw`_
|
`pyexcel-io`_ `xls`_ `xlsx`_ `ods`_ `ods3`_ `odsr`_ `xlsxw`_
|
||||||
============= ======= ======== ======= ======== ======== ========
|
============= ======= ======== ======= ======== ======== ========
|
||||||
0.5.1 0.5.0 0.5.0 0.5.0 0.5.0 0.5.0 0.5.0
|
0.6.0+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+
|
||||||
|
0.5.10+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+
|
||||||
|
0.5.1+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+
|
||||||
0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x
|
0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x
|
||||||
0.3.0+ 0.3.0+ 0.3.0 0.3.0+ 0.3.0+ 0.3.0 0.3.0
|
0.3.0+ 0.3.0+ 0.3.0 0.3.0+ 0.3.0+ 0.3.0 0.3.0
|
||||||
0.2.2+ 0.2.2+ 0.2.2+ 0.2.1+ 0.2.1+ 0.0.1
|
0.2.2+ 0.2.2+ 0.2.2+ 0.2.1+ 0.2.1+ 0.0.1
|
||||||
|
@ -106,6 +108,7 @@ API
|
||||||
.. autosummary::
|
.. autosummary::
|
||||||
:toctree: api/
|
:toctree: api/
|
||||||
|
|
||||||
|
iget_data
|
||||||
get_data
|
get_data
|
||||||
save_data
|
save_data
|
||||||
|
|
|
@ -6,4 +6,3 @@
|
||||||
{%block pyexcel_extra_classifiers%}
|
{%block pyexcel_extra_classifiers%}
|
||||||
'Programming Language :: Python :: Implementation :: PyPy'
|
'Programming Language :: Python :: Implementation :: PyPy'
|
||||||
{%endblock%}}
|
{%endblock%}}
|
||||||
|
|
|
@ -1,6 +0,0 @@
|
||||||
{% extends 'tests/requirements.txt.jj2' %}
|
|
||||||
{%block extras %}
|
|
||||||
SQLAlchemy
|
|
||||||
pyexcel>=0.2.0
|
|
||||||
pyexcel-xls>=0.1.0
|
|
||||||
{%endblock%}
|
|
29
.moban.yml
29
.moban.yml
|
@ -1,26 +1,9 @@
|
||||||
requires:
|
overrides: "git://github.com/pyexcel/pyexcel-mobans!/mobanfile.yaml"
|
||||||
- type: git
|
|
||||||
url: https://github.com/moremoban/pypi-mobans
|
|
||||||
submodule: true
|
|
||||||
- https://github.com/pyexcel/pyexcel-mobans
|
|
||||||
configuration:
|
configuration:
|
||||||
configuration_dir: "pyexcel-mobans:config"
|
|
||||||
template_dir:
|
|
||||||
- "pyexcel-mobans:templates"
|
|
||||||
- "pypi-mobans:templates"
|
|
||||||
- ".moban.d"
|
|
||||||
configuration: pyexcel-io.yml
|
configuration: pyexcel-io.yml
|
||||||
targets:
|
targets:
|
||||||
- "docs/source/conf.py": "docs/source/conf.py"
|
- "docs/source/conf.py": "docs/source/custom_conf.py.jj2"
|
||||||
- setup.py: setup.py
|
- setup.py: io_setup.py.jj2
|
||||||
- .travis.yml: travis.yml.jj2
|
- README.rst: io_readme.rst.jj2
|
||||||
- requirements.txt: requirements.txt.jj2
|
- "docs/source/index.rst": "docs/source/index.rst.jj2"
|
||||||
- "tests/requirements.txt": "tests/requirements.txt"
|
- .gitignore: gitignore.jj2
|
||||||
- LICENSE: NEW_BSD_LICENSE.jj2
|
|
||||||
- test.sh: test.script.jj2
|
|
||||||
- test.bat: test.script.jj2
|
|
||||||
- README.rst: README.rst
|
|
||||||
- "docs/source/index.rst": "docs/source/index.rst"
|
|
||||||
- output: CHANGELOG.rst
|
|
||||||
configuration: changelog.yml
|
|
||||||
template: CHANGELOG.rst.jj2
|
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
# .readthedocs.yml
|
||||||
|
# Read the Docs configuration file
|
||||||
|
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
|
||||||
|
|
||||||
|
# Required
|
||||||
|
version: 2
|
||||||
|
|
||||||
|
# Build documentation in the docs/ directory with Sphinx
|
||||||
|
sphinx:
|
||||||
|
configuration: docs/source/conf.py
|
||||||
|
|
||||||
|
# Optionally build your docs in additional formats such as PDF
|
||||||
|
formats:
|
||||||
|
- pdf
|
||||||
|
|
||||||
|
# Optionally set the version of Python and requirements required to build your docs
|
||||||
|
python:
|
||||||
|
version: 3.7
|
23
.travis.yml
23
.travis.yml
|
@ -1,23 +0,0 @@
|
||||||
sudo: false
|
|
||||||
language: python
|
|
||||||
notifications:
|
|
||||||
email: false
|
|
||||||
python:
|
|
||||||
- pypy-5.3.1
|
|
||||||
- 3.7-dev
|
|
||||||
- 3.6
|
|
||||||
- 3.5
|
|
||||||
- 3.4
|
|
||||||
- 2.7
|
|
||||||
before_install:
|
|
||||||
- if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi
|
|
||||||
- if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then
|
|
||||||
mv min_requirements.txt requirements.txt ;
|
|
||||||
fi
|
|
||||||
- test ! -f rnd_requirements.txt || pip install --no-deps -r rnd_requirements.txt
|
|
||||||
- test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ;
|
|
||||||
- pip install -r tests/requirements.txt
|
|
||||||
script:
|
|
||||||
- make test
|
|
||||||
after_success:
|
|
||||||
codecov
|
|
240
CHANGELOG.rst
240
CHANGELOG.rst
|
@ -1,11 +1,134 @@
|
||||||
Change log
|
Change log
|
||||||
================================================================================
|
================================================================================
|
||||||
|
|
||||||
|
0.6.6 - 31.1.2022
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#112 <https://github.com/pyexcel/pyexcel-io/issues/112>`_: Log Empty Row
|
||||||
|
Warning instead 'print'
|
||||||
|
|
||||||
|
0.6.5 - 08.10.2021
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#109 <https://github.com/pyexcel/pyexcel-io/issues/109>`_: enable ods3 to
|
||||||
|
have datetime
|
||||||
|
|
||||||
|
0.6.4 - 31.10.2020
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#102 <https://github.com/pyexcel/pyexcel-io/issues/102>`_: skip columns from
|
||||||
|
imported excel sheet.
|
||||||
|
|
||||||
|
0.6.3 - 12.10.2020
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**fixed**
|
||||||
|
|
||||||
|
#. `#96 <https://github.com/pyexcel/pyexcel-io/issues/96>`_: regression: unknown
|
||||||
|
file type shall trigger NoSupportingPluginFound
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. extra dependencies uses 0.6.0 based plugins
|
||||||
|
|
||||||
|
0.6.2 - 7.10.2020
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#94 <https://github.com/pyexcel/pyexcel-io/issues/94>`_: keep backward
|
||||||
|
compatibility for pyexcel-xls 0.4.1
|
||||||
|
|
||||||
|
0.6.1 - 7.10.2020
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**removed**
|
||||||
|
|
||||||
|
#. python 3.6 lower versions are no longer supported
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. pyexcel-io plugin interface has been rewritten. PyInstaller user will be
|
||||||
|
impacted. please read 'Packaging with Pyinstaller' in the documentation.
|
||||||
|
#. new query set reader plugin. pyexcel<=0.6.4 has used intrusive way of getting
|
||||||
|
query set source done. it is against the plugin interface.
|
||||||
|
|
||||||
|
**fixed**
|
||||||
|
|
||||||
|
#. `#74 <https://github.com/pyexcel/pyexcel-io/issues/74>`_: handle zip files
|
||||||
|
which contain non-UTF-8 encoded files.
|
||||||
|
|
||||||
|
**added**
|
||||||
|
|
||||||
|
#. `#86 <https://github.com/pyexcel/pyexcel-io/issues/86>`_: allow trailing
|
||||||
|
options, get_data(...keep_trailing_empty_cells=True).
|
||||||
|
|
||||||
|
0.5.20 - 17.7.2019
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#70 <https://github.com/pyexcel/pyexcel-io/issues/70>`_: when the given file
|
||||||
|
is a root directory, the error shall read it is not a file
|
||||||
|
|
||||||
|
0.5.19 - 14.7.2019
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `pyexcel#185 <https://github.com/pyexcel/pyexcel/issues/185>`_: handle stream
|
||||||
|
conversion if file type(html) needs string content then bytes to handle
|
||||||
|
|
||||||
|
0.5.18 - 12.06.2019
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#69 <https://github.com/pyexcel/pyexcel-io/issues/69>`_: Force file
|
||||||
|
type(force_file_type) on write
|
||||||
|
|
||||||
|
0.5.17 - 04.04.2019
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#68 <https://github.com/pyexcel/pyexcel-io/issues/68>`_: Raise IOError when
|
||||||
|
the data file does not exist
|
||||||
|
|
||||||
|
0.5.16 - 19.03.2019
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#67 <https://github.com/pyexcel/pyexcel-io/issues/67>`_: fix conversion
|
||||||
|
issue for long type on python 2.7 for ods
|
||||||
|
|
||||||
|
0.5.15 - 16.03.2019
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `pyexcel-ods#33 <https://github.com/pyexcel/pyexcel-ods/issues/33>`_: fix
|
||||||
|
integer comparision error on i586
|
||||||
|
|
||||||
|
0.5.14 - 21.02.2019
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
**updated**
|
||||||
|
|
||||||
|
#. `#65 <https://github.com/pyexcel/pyexcel-io/issues/65>`_: add
|
||||||
|
tests/__init__.py because python2.7 setup.py test needs it
|
||||||
|
|
||||||
0.5.13 - 12.02.2019
|
0.5.13 - 12.02.2019
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
updated
|
**updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#63 <https://github.com/pyexcel/pyexcel-io/issues/63>`_: Version 0.5.12
|
#. `#63 <https://github.com/pyexcel/pyexcel-io/issues/63>`_: Version 0.5.12
|
||||||
prevents xslx and ods plugin from being loaded
|
prevents xslx and ods plugin from being loaded
|
||||||
|
@ -13,19 +136,17 @@ updated
|
||||||
0.5.12 - 9.02.2019
|
0.5.12 - 9.02.2019
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
updated
|
**updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#60 <https://github.com/pyexcel/pyexcel-io/issues/60>`_: include tests in
|
#. `#60 <https://github.com/pyexcel/pyexcel-io/issues/60>`_: include tests in
|
||||||
tar ball
|
tar ball
|
||||||
#. `#61 <https://github.com/pyexcel/pyexcel-io/issues/61>`_: enable python
|
#. `#61 <https://github.com/pyexcel/pyexcel-io/issues/61>`_: enable python
|
||||||
setup.py test
|
setup.py test
|
||||||
|
|
||||||
0.5.10 - 3.12.2018
|
0.5.11 - 3.12.2018
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
updated
|
**updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#59 <https://github.com/pyexcel/pyexcel-io/issues/59>`_: Please use
|
#. `#59 <https://github.com/pyexcel/pyexcel-io/issues/59>`_: Please use
|
||||||
scan_plugins_regex, which lml 0.7 complains about
|
scan_plugins_regex, which lml 0.7 complains about
|
||||||
|
@ -33,8 +154,7 @@ updated
|
||||||
0.5.10 - 27.11.2018
|
0.5.10 - 27.11.2018
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
added
|
**added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#57 <https://github.com/pyexcel/pyexcel-io/issues/57>`_, long type will not
|
#. `#57 <https://github.com/pyexcel/pyexcel-io/issues/57>`_, long type will not
|
||||||
be written in ods. please use string type. And if the integer is equal or
|
be written in ods. please use string type. And if the integer is equal or
|
||||||
|
@ -45,8 +165,7 @@ added
|
||||||
0.5.9.1 - 30.08.2018
|
0.5.9.1 - 30.08.2018
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
updated
|
**updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#53 <https://github.com/pyexcel/pyexcel-io/issues/53>`_, upgrade lml
|
#. `#53 <https://github.com/pyexcel/pyexcel-io/issues/53>`_, upgrade lml
|
||||||
dependency to at least 0.0.2
|
dependency to at least 0.0.2
|
||||||
|
@ -54,8 +173,7 @@ updated
|
||||||
0.5.9 - 23.08.2018
|
0.5.9 - 23.08.2018
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
added
|
**added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `pyexcel#148 <https://github.com/pyexcel/pyexcel/issues/148>`_, support
|
#. `pyexcel#148 <https://github.com/pyexcel/pyexcel/issues/148>`_, support
|
||||||
force_file_type
|
force_file_type
|
||||||
|
@ -63,8 +181,7 @@ added
|
||||||
0.5.8 - 16.08.2018
|
0.5.8 - 16.08.2018
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
added
|
**added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#49 <https://github.com/pyexcel/pyexcel-io/issues/49>`_, support additional
|
#. `#49 <https://github.com/pyexcel/pyexcel-io/issues/49>`_, support additional
|
||||||
options when detecting float values in csv format. default_float_nan,
|
options when detecting float values in csv format. default_float_nan,
|
||||||
|
@ -73,8 +190,7 @@ added
|
||||||
0.5.7 - 02.05.2018
|
0.5.7 - 02.05.2018
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
fixed
|
**fixed**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#48 <https://github.com/pyexcel/pyexcel-io/issues/48>`_, turn off pep 0515
|
#. `#48 <https://github.com/pyexcel/pyexcel-io/issues/48>`_, turn off pep 0515
|
||||||
#. `#47 <https://github.com/pyexcel/pyexcel-io/issues/47>`_, csv reader cannot
|
#. `#47 <https://github.com/pyexcel/pyexcel-io/issues/47>`_, csv reader cannot
|
||||||
|
@ -83,8 +199,7 @@ fixed
|
||||||
0.5.6 - 11.01.2018
|
0.5.6 - 11.01.2018
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
fixed
|
**fixed**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#46 <https://github.com/pyexcel/pyexcel-io/issues/46>`_, expose `bulk_save`
|
#. `#46 <https://github.com/pyexcel/pyexcel-io/issues/46>`_, expose `bulk_save`
|
||||||
to developer
|
to developer
|
||||||
|
@ -92,8 +207,7 @@ fixed
|
||||||
0.5.5 - 23.12.2017
|
0.5.5 - 23.12.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
fixed
|
**fixed**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. Issue `#45 <https://github.com/pyexcel/pyexcel-io/issues/45>`_, csv reader
|
#. Issue `#45 <https://github.com/pyexcel/pyexcel-io/issues/45>`_, csv reader
|
||||||
throws exception because google app engine does not support mmap. People who
|
throws exception because google app engine does not support mmap. People who
|
||||||
|
@ -103,8 +217,7 @@ fixed
|
||||||
0.5.4 - 10.11.2017
|
0.5.4 - 10.11.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
updated
|
**updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. PR `#44 <https://github.com/pyexcel/pyexcel-io/pull/44>`_, use unicodewriter
|
#. PR `#44 <https://github.com/pyexcel/pyexcel-io/pull/44>`_, use unicodewriter
|
||||||
for csvz writers.
|
for csvz writers.
|
||||||
|
@ -112,8 +225,7 @@ updated
|
||||||
0.5.3 - 23.10.2017
|
0.5.3 - 23.10.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
updated
|
**updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. pyexcel `pyexcel#105 <https://github.com/pyexcel/pyexcel/issues/105>`_,
|
#. pyexcel `pyexcel#105 <https://github.com/pyexcel/pyexcel/issues/105>`_,
|
||||||
remove gease from setup_requires, introduced by 0.5.2.
|
remove gease from setup_requires, introduced by 0.5.2.
|
||||||
|
@ -122,8 +234,7 @@ updated
|
||||||
0.5.2 - 20.10.2017
|
0.5.2 - 20.10.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
added
|
**added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `pyexcel#103 <https://github.com/pyexcel/pyexcel/issues/103>`_, include
|
#. `pyexcel#103 <https://github.com/pyexcel/pyexcel/issues/103>`_, include
|
||||||
LICENSE file in MANIFEST.in, meaning LICENSE file will appear in the released
|
LICENSE file in MANIFEST.in, meaning LICENSE file will appear in the released
|
||||||
|
@ -132,8 +243,7 @@ added
|
||||||
0.5.1 - 02.09.2017
|
0.5.1 - 02.09.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Fixed
|
**Fixed**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `pyexcel-ods#25 <https://github.com/pyexcel/pyexcel-ods/issues/25>`_,
|
#. `pyexcel-ods#25 <https://github.com/pyexcel/pyexcel-ods/issues/25>`_,
|
||||||
Unwanted dependency on pyexcel.
|
Unwanted dependency on pyexcel.
|
||||||
|
@ -141,13 +251,11 @@ Fixed
|
||||||
0.5.0 - 30.08.2017
|
0.5.0 - 30.08.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Added
|
**Added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. Collect all data type conversion codes as service.py.
|
#. Collect all data type conversion codes as service.py.
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#19 <https://github.com/pyexcel/pyexcel-io/issues/19>`_, use cString by
|
#. `#19 <https://github.com/pyexcel/pyexcel-io/issues/19>`_, use cString by
|
||||||
default. For python, it will be a performance boost
|
default. For python, it will be a performance boost
|
||||||
|
@ -155,8 +263,7 @@ Updated
|
||||||
0.4.4 - 08.08.2017
|
0.4.4 - 08.08.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#42 <https://github.com/pyexcel/pyexcel-io/issues/42>`_, raise exception if
|
#. `#42 <https://github.com/pyexcel/pyexcel-io/issues/42>`_, raise exception if
|
||||||
database table name does not match the sheet name
|
database table name does not match the sheet name
|
||||||
|
@ -164,8 +271,7 @@ Updated
|
||||||
0.4.3 - 29.07.2017
|
0.4.3 - 29.07.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#41 <https://github.com/pyexcel/pyexcel-io/issues/41>`_, walk away
|
#. `#41 <https://github.com/pyexcel/pyexcel-io/issues/41>`_, walk away
|
||||||
gracefully when mmap is not available.
|
gracefully when mmap is not available.
|
||||||
|
@ -173,8 +279,7 @@ Updated
|
||||||
0.4.2 - 05.07.2017
|
0.4.2 - 05.07.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#37 <https://github.com/pyexcel/pyexcel-io/issues/37>`_, permanently fix the
|
#. `#37 <https://github.com/pyexcel/pyexcel-io/issues/37>`_, permanently fix the
|
||||||
residue folder pyexcel by release all future releases in a clean clone.
|
residue folder pyexcel by release all future releases in a clean clone.
|
||||||
|
@ -182,8 +287,7 @@ Updated
|
||||||
0.4.1 - 29.06.2017
|
0.4.1 - 29.06.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#39 <https://github.com/pyexcel/pyexcel-io/issues/39>`_, raise exception
|
#. `#39 <https://github.com/pyexcel/pyexcel-io/issues/39>`_, raise exception
|
||||||
when bulk save in django fails. Please `bulk_save=False` if you as the
|
when bulk save in django fails. Please `bulk_save=False` if you as the
|
||||||
|
@ -195,8 +299,7 @@ Updated
|
||||||
0.4.0 - 19.06.2017
|
0.4.0 - 19.06.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. 'built-in' as the value to the parameter 'library' as parameter to invoke
|
#. 'built-in' as the value to the parameter 'library' as parameter to invoke
|
||||||
pyexcel-io's built-in csv, tsv, csvz, tsvz, django and sql won't work. It is
|
pyexcel-io's built-in csv, tsv, csvz, tsvz, django and sql won't work. It is
|
||||||
|
@ -207,16 +310,14 @@ Updated
|
||||||
handle are made sure to be closed. File close mechanism is enfored.
|
handle are made sure to be closed. File close mechanism is enfored.
|
||||||
#. iget_data function is introduced to cope with dangling file handle issue.
|
#. iget_data function is introduced to cope with dangling file handle issue.
|
||||||
|
|
||||||
Removed
|
**Removed**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. Removed plugin loading code and lml is used instead
|
#. Removed plugin loading code and lml is used instead
|
||||||
|
|
||||||
0.3.4 - 18.05.2017
|
0.3.4 - 18.05.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#33 <https://github.com/pyexcel/pyexcel-io/issues/33>`_, handle mmap object
|
#. `#33 <https://github.com/pyexcel/pyexcel-io/issues/33>`_, handle mmap object
|
||||||
differently given as file content. This issue has put in a priority to single
|
differently given as file content. This issue has put in a priority to single
|
||||||
|
@ -232,16 +333,14 @@ Updated
|
||||||
0.3.3 - 30.03.2017
|
0.3.3 - 30.03.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#31 <https://github.com/pyexcel/pyexcel-io/issues/31>`_, support pyinstaller
|
#. `#31 <https://github.com/pyexcel/pyexcel-io/issues/31>`_, support pyinstaller
|
||||||
|
|
||||||
0.3.2 - 26.01.2017
|
0.3.2 - 26.01.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#29 <https://github.com/pyexcel/pyexcel-io/issues/29>`_, change
|
#. `#29 <https://github.com/pyexcel/pyexcel-io/issues/29>`_, change
|
||||||
skip_empty_rows to False by default
|
skip_empty_rows to False by default
|
||||||
|
@ -249,13 +348,11 @@ Updated
|
||||||
0.3.1 - 21.01.2017
|
0.3.1 - 21.01.2017
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Added
|
**Added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. updated versions of extra packages
|
#. updated versions of extra packages
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#23 <https://github.com/pyexcel/pyexcel-io/issues/23>`_, provide helpful
|
#. `#23 <https://github.com/pyexcel/pyexcel-io/issues/23>`_, provide helpful
|
||||||
message when old pyexcel plugin exists
|
message when old pyexcel plugin exists
|
||||||
|
@ -264,8 +361,7 @@ Updated
|
||||||
0.3.0 - 22.12.2016
|
0.3.0 - 22.12.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Added
|
**Added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. lazy loading of plugins. for example, pyexcel-xls is not entirely loaded
|
#. lazy loading of plugins. for example, pyexcel-xls is not entirely loaded
|
||||||
until xls format is used at its first attempted reading or writing. Since it
|
until xls format is used at its first attempted reading or writing. Since it
|
||||||
|
@ -276,16 +372,14 @@ Added
|
||||||
0.2.6 - 21.12.2016
|
0.2.6 - 21.12.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#24 <https://github.com/pyexcel/pyexcel-io/issues/24>`__, pass on batch_size
|
#. `#24 <https://github.com/pyexcel/pyexcel-io/issues/24>`__, pass on batch_size
|
||||||
|
|
||||||
0.2.5 - 20.12.2016
|
0.2.5 - 20.12.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#26 <https://github.com/pyexcel/pyexcel-io/issues/26>`__, performance issue
|
#. `#26 <https://github.com/pyexcel/pyexcel-io/issues/26>`__, performance issue
|
||||||
with getting the number of columns.
|
with getting the number of columns.
|
||||||
|
@ -293,8 +387,7 @@ Updated
|
||||||
0.2.4 - 24.11.2016
|
0.2.4 - 24.11.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#23 <https://github.com/pyexcel/pyexcel-io/issues/23>`__, Failed to convert
|
#. `#23 <https://github.com/pyexcel/pyexcel-io/issues/23>`__, Failed to convert
|
||||||
long integer string in python 2 to its actual value
|
long integer string in python 2 to its actual value
|
||||||
|
@ -302,8 +395,7 @@ Updated
|
||||||
0.2.3 - 16.09.2016
|
0.2.3 - 16.09.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Added
|
**Added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#21 <https://github.com/pyexcel/pyexcel-io/issues/21>`__, choose subset from
|
#. `#21 <https://github.com/pyexcel/pyexcel-io/issues/21>`__, choose subset from
|
||||||
data base tables for export
|
data base tables for export
|
||||||
|
@ -313,16 +405,14 @@ Added
|
||||||
0.2.2 - 31.08.2016
|
0.2.2 - 31.08.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Added
|
**Added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. support pagination. two pairs: start_row, row_limit and start_column,
|
#. support pagination. two pairs: start_row, row_limit and start_column,
|
||||||
column_limit help you deal with large files.
|
column_limit help you deal with large files.
|
||||||
#. `skip_empty_rows=True` was introduced. To include empty rows, put it to
|
#. `skip_empty_rows=True` was introduced. To include empty rows, put it to
|
||||||
False.
|
False.
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. `#20 <https://github.com/pyexcel/pyexcel-io/issues/20>`__, pyexcel-io
|
#. `#20 <https://github.com/pyexcel/pyexcel-io/issues/20>`__, pyexcel-io
|
||||||
attempts to parse cell contents of 'infinity' as a float/int, crashes
|
attempts to parse cell contents of 'infinity' as a float/int, crashes
|
||||||
|
@ -330,8 +420,7 @@ Updated
|
||||||
0.2.1 - 11.07.2016
|
0.2.1 - 11.07.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Added
|
**Added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. csv format: handle utf-16 encoded csv files. Potentially being able to decode
|
#. csv format: handle utf-16 encoded csv files. Potentially being able to decode
|
||||||
other formats if correct "encoding" is provided
|
other formats if correct "encoding" is provided
|
||||||
|
@ -339,8 +428,7 @@ Added
|
||||||
supported
|
supported
|
||||||
#. support stdin as input stream and stdout as output stream
|
#. support stdin as input stream and stdout as output stream
|
||||||
|
|
||||||
Updated
|
**Updated**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. Attention, user of pyexcel-io! No longer io stream validation is performed in
|
#. Attention, user of pyexcel-io! No longer io stream validation is performed in
|
||||||
python 3. The guideline is: io.StringIO for csv, tsv only, otherwise BytesIO
|
python 3. The guideline is: io.StringIO for csv, tsv only, otherwise BytesIO
|
||||||
|
@ -352,8 +440,7 @@ Updated
|
||||||
0.2.0 - 01.06.2016
|
0.2.0 - 01.06.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Added
|
**Added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. autoload of pyexcel-io plugins
|
#. autoload of pyexcel-io plugins
|
||||||
#. auto detect `datetime`, `float` and `int`. Detection can be switched off by
|
#. auto detect `datetime`, `float` and `int`. Detection can be switched off by
|
||||||
|
@ -362,7 +449,6 @@ Added
|
||||||
0.1.0 - 17.01.2016
|
0.1.0 - 17.01.2016
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
Added
|
**Added**
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
#. yield key word to return generator as content
|
#. yield key word to return generator as content
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
|
||||||
|
|
||||||
|
6 contributors
|
||||||
|
================================================================================
|
||||||
|
|
||||||
|
In alphabetical order:
|
||||||
|
|
||||||
|
* `Craig Anderson <https://github.com/craiga>`_
|
||||||
|
* `John Vandenberg <https://github.com/jayvdb>`_
|
||||||
|
* `Stephen J. Fuhry <https://github.com/fuhrysteve>`_
|
||||||
|
* `Stephen Rauch <https://github.com/stephenrauch>`_
|
||||||
|
* `Vincent Raspal <https://github.com/vinraspa>`_
|
||||||
|
* `Víctor Antonio Hernández Monroy <https://github.com/antherkiv>`_
|
4
LICENSE
4
LICENSE
|
@ -1,4 +1,4 @@
|
||||||
Copyright (c) 2015-2019 by Onni Software Ltd. and its contributors
|
Copyright (c) 2015-2022 by Onni Software Ltd. and its contributors
|
||||||
All rights reserved.
|
All rights reserved.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms of the software as well
|
Redistribution and use in source and binary forms of the software as well
|
||||||
|
@ -13,7 +13,7 @@ that the following conditions are met:
|
||||||
and/or other materials provided with the distribution.
|
and/or other materials provided with the distribution.
|
||||||
|
|
||||||
* Neither the name of 'pyexcel-io' nor the names of the contributors
|
* Neither the name of 'pyexcel-io' nor the names of the contributors
|
||||||
may be used to endorse or promote products derived from this software
|
may not be used to endorse or promote products derived from this software
|
||||||
without specific prior written permission.
|
without specific prior written permission.
|
||||||
|
|
||||||
THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
|
THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
include README.rst
|
include README.rst
|
||||||
include LICENSE
|
include LICENSE
|
||||||
include CHANGELOG.rst
|
include CHANGELOG.rst
|
||||||
|
include CONTRIBUTORS.rst
|
||||||
recursive-include tests *
|
recursive-include tests *
|
||||||
include docs/source/*
|
recursive-include docs *
|
||||||
include Makefile
|
include Makefile
|
||||||
include test.sh
|
include test.sh
|
||||||
|
|
||||||
|
|
16
Makefile
16
Makefile
|
@ -1,8 +1,16 @@
|
||||||
all: test
|
all: test
|
||||||
|
|
||||||
test:
|
test: lint
|
||||||
bash test.sh
|
bash test.sh
|
||||||
|
|
||||||
document:
|
install_test:
|
||||||
sphinx-autogen -o docs/source/generated/ docs/source/*.rst
|
pip install -r tests/requirements.txt
|
||||||
sphinx-build -b html docs/source/ docs/build/
|
|
||||||
|
lint:
|
||||||
|
bash lint.sh
|
||||||
|
|
||||||
|
format:
|
||||||
|
bash format.sh
|
||||||
|
|
||||||
|
git-diff-check:
|
||||||
|
git diff --exit-code
|
||||||
|
|
161
README.rst
161
README.rst
|
@ -3,20 +3,37 @@ pyexcel-io - Let you focus on data, instead of file formats
|
||||||
================================================================================
|
================================================================================
|
||||||
|
|
||||||
.. image:: https://raw.githubusercontent.com/pyexcel/pyexcel.github.io/master/images/patreon.png
|
.. image:: https://raw.githubusercontent.com/pyexcel/pyexcel.github.io/master/images/patreon.png
|
||||||
:target: https://www.patreon.com/pyexcel
|
:target: https://www.patreon.com/chfw
|
||||||
|
|
||||||
.. image:: https://api.bountysource.com/badge/team?team_id=288537
|
.. image:: https://raw.githubusercontent.com/pyexcel/pyexcel-mobans/master/images/awesome-badge.svg
|
||||||
:target: https://salt.bountysource.com/teams/chfw-pyexcel
|
:target: https://awesome-python.com/#specific-formats-processing
|
||||||
|
|
||||||
.. image:: https://travis-ci.org/pyexcel/pyexcel-io.svg?branch=master
|
.. image:: https://github.com/pyexcel/pyexcel-io/workflows/run_tests/badge.svg
|
||||||
:target: http://travis-ci.org/pyexcel/pyexcel-io
|
:target: http://github.com/pyexcel/pyexcel-io/actions
|
||||||
|
|
||||||
.. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg
|
.. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg
|
||||||
:target: https://codecov.io/gh/pyexcel/pyexcel-io
|
:target: https://codecov.io/gh/pyexcel/pyexcel-io
|
||||||
|
|
||||||
|
.. image:: https://badge.fury.io/py/pyexcel-io.svg
|
||||||
|
:target: https://pypi.org/project/pyexcel-io
|
||||||
|
|
||||||
|
.. image:: https://anaconda.org/conda-forge/pyexcel-io/badges/version.svg
|
||||||
|
:target: https://anaconda.org/conda-forge/pyexcel-io
|
||||||
|
|
||||||
|
.. image:: https://pepy.tech/badge/pyexcel-io/month
|
||||||
|
:target: https://pepy.tech/project/pyexcel-io
|
||||||
|
|
||||||
|
.. image:: https://anaconda.org/conda-forge/pyexcel-io/badges/downloads.svg
|
||||||
|
:target: https://anaconda.org/conda-forge/pyexcel-io
|
||||||
|
|
||||||
.. image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg
|
.. image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg
|
||||||
:target: https://gitter.im/pyexcel/Lobby
|
:target: https://gitter.im/pyexcel/Lobby
|
||||||
|
|
||||||
|
.. image:: https://img.shields.io/static/v1?label=continuous%20templating&message=%E6%A8%A1%E7%89%88%E6%9B%B4%E6%96%B0&color=blue&style=flat-square
|
||||||
|
:target: https://moban.readthedocs.io/en/latest/#at-scale-continous-templating-for-open-source-projects
|
||||||
|
|
||||||
|
.. image:: https://img.shields.io/static/v1?label=coding%20style&message=black&color=black&style=flat-square
|
||||||
|
:target: https://github.com/psf/black
|
||||||
.. image:: https://readthedocs.org/projects/pyexcel-io/badge/?version=latest
|
.. image:: https://readthedocs.org/projects/pyexcel-io/badge/?version=latest
|
||||||
:target: http://pyexcel-io.readthedocs.org/en/latest/
|
:target: http://pyexcel-io.readthedocs.org/en/latest/
|
||||||
|
|
||||||
|
@ -24,7 +41,7 @@ Support the project
|
||||||
================================================================================
|
================================================================================
|
||||||
|
|
||||||
If your company has embedded pyexcel and its components into a revenue generating
|
If your company has embedded pyexcel and its components into a revenue generating
|
||||||
product, please support me on `patreon <https://www.patreon.com/bePatron?u=5537627>`_
|
product, please support me on github, `patreon <https://www.patreon.com/bePatron?u=5537627>`_
|
||||||
or `bounty source <https://salt.bountysource.com/teams/chfw-pyexcel>`_ to maintain
|
or `bounty source <https://salt.bountysource.com/teams/chfw-pyexcel>`_ to maintain
|
||||||
the project and develop it further.
|
the project and develop it further.
|
||||||
|
|
||||||
|
@ -43,6 +60,8 @@ Known constraints
|
||||||
|
|
||||||
Fonts, colors and charts are not supported.
|
Fonts, colors and charts are not supported.
|
||||||
|
|
||||||
|
Nor to read password protected xls, xlsx and ods files.
|
||||||
|
|
||||||
Introduction
|
Introduction
|
||||||
================================================================================
|
================================================================================
|
||||||
|
|
||||||
|
@ -57,48 +76,47 @@ sqlalchemy supported databases. Its supported file formats are extended to cover
|
||||||
|
|
||||||
.. table:: A list of file formats supported by external plugins
|
.. table:: A list of file formats supported by external plugins
|
||||||
|
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
Package name Supported file formats Dependencies Python versions
|
Package name Supported file formats Dependencies
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
`pyexcel-io`_ csv, csvz [#f1]_, tsv, 2.6, 2.7, 3.3,
|
`pyexcel-io`_ csv, csvz [#f1]_, tsv,
|
||||||
tsvz [#f2]_ 3.4, 3.5, 3.6
|
tsvz [#f2]_
|
||||||
pypy
|
`pyexcel-xls`_ xls, xlsx(read only), `xlrd`_,
|
||||||
`pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, same as above
|
|
||||||
xlsm(read only) `xlwt`_
|
xlsm(read only) `xlwt`_
|
||||||
`pyexcel-xlsx`_ xlsx `openpyxl`_ same as above
|
`pyexcel-xlsx`_ xlsx `openpyxl`_
|
||||||
`pyexcel-ods3`_ ods `pyexcel-ezodf`_, 2.6, 2.7, 3.3, 3.4
|
`pyexcel-ods3`_ ods `pyexcel-ezodf`_,
|
||||||
lxml 3.5, 3.6
|
lxml
|
||||||
`pyexcel-ods`_ ods `odfpy`_ same as above
|
`pyexcel-ods`_ ods `odfpy`_
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
|
|
||||||
.. table:: Dedicated file reader and writers
|
.. table:: Dedicated file reader and writers
|
||||||
|
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
Package name Supported file formats Dependencies Python versions
|
Package name Supported file formats Dependencies
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
`pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ Python 2 and 3
|
`pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_
|
||||||
`pyexcel-xlsxr`_ xlsx(read only) lxml same as above
|
`pyexcel-libxlsxw`_ xlsx(write only) `libxlsxwriter`_
|
||||||
`pyexcel-odsr`_ read only for ods, fods lxml same as above
|
`pyexcel-xlsxr`_ xlsx(read only) lxml
|
||||||
`pyexcel-htmlr`_ html(read only) lxml,html5lib same as above
|
`pyexcel-xlsbr`_ xlsb(read only) pyxlsb
|
||||||
======================== ======================= ================= ==================
|
`pyexcel-odsr`_ read only for ods, fods lxml
|
||||||
|
`pyexcel-odsw`_ write only for ods loxun
|
||||||
|
`pyexcel-htmlr`_ html(read only) lxml,html5lib
|
||||||
|
`pyexcel-pdfr`_ pdf(read only) camelot
|
||||||
|
======================== ======================= =================
|
||||||
|
|
||||||
|
|
||||||
.. _pyexcel-io: https://github.com/pyexcel/pyexcel-io
|
Plugin shopping guide
|
||||||
.. _pyexcel-xls: https://github.com/pyexcel/pyexcel-xls
|
------------------------
|
||||||
.. _pyexcel-xlsx: https://github.com/pyexcel/pyexcel-xlsx
|
|
||||||
.. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods
|
|
||||||
.. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3
|
|
||||||
.. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr
|
|
||||||
.. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw
|
|
||||||
.. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr
|
|
||||||
.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr
|
|
||||||
|
|
||||||
.. _xlrd: https://github.com/python-excel/xlrd
|
Since 2020, all pyexcel-io plugins have dropped the support for python versions
|
||||||
.. _xlwt: https://github.com/python-excel/xlwt
|
which are lower than 3.6. If you want to use any of those Python versions, please use pyexcel-io
|
||||||
.. _openpyxl: https://bitbucket.org/openpyxl/openpyxl
|
and its plugins versions that are lower than 0.6.0.
|
||||||
.. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter
|
|
||||||
.. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf
|
|
||||||
.. _odfpy: https://github.com/eea/odfpy
|
Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of
|
||||||
|
xml files
|
||||||
|
|
||||||
|
The dedicated readers for excel files can stream read
|
||||||
|
|
||||||
|
|
||||||
In order to manage the list of plugins installed, you need to use pip to add or remove
|
In order to manage the list of plugins installed, you need to use pip to add or remove
|
||||||
|
@ -108,6 +126,32 @@ in your environment, you need to tell pyexcel which plugin to use per function c
|
||||||
For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr.
|
For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr.
|
||||||
You need to append get_array(..., library='pyexcel-odsr').
|
You need to append get_array(..., library='pyexcel-odsr').
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
.. _pyexcel-io: https://github.com/pyexcel/pyexcel-io
|
||||||
|
.. _pyexcel-xls: https://github.com/pyexcel/pyexcel-xls
|
||||||
|
.. _pyexcel-xlsx: https://github.com/pyexcel/pyexcel-xlsx
|
||||||
|
.. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods
|
||||||
|
.. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3
|
||||||
|
.. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr
|
||||||
|
.. _pyexcel-odsw: https://github.com/pyexcel/pyexcel-odsw
|
||||||
|
.. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr
|
||||||
|
|
||||||
|
.. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw
|
||||||
|
.. _pyexcel-libxlsxw: https://github.com/pyexcel/pyexcel-libxlsxw
|
||||||
|
.. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr
|
||||||
|
.. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr
|
||||||
|
.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr
|
||||||
|
|
||||||
|
.. _xlrd: https://github.com/python-excel/xlrd
|
||||||
|
.. _xlwt: https://github.com/python-excel/xlwt
|
||||||
|
.. _openpyxl: https://bitbucket.org/openpyxl/openpyxl
|
||||||
|
.. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter
|
||||||
|
.. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf
|
||||||
|
.. _odfpy: https://github.com/eea/odfpy
|
||||||
|
.. _libxlsxwriter: http://libxlsxwriter.github.io/getting_started.html
|
||||||
|
|
||||||
|
|
||||||
.. rubric:: Footnotes
|
.. rubric:: Footnotes
|
||||||
|
|
||||||
.. [#f1] zipped csv file
|
.. [#f1] zipped csv file
|
||||||
|
@ -161,7 +205,7 @@ Then install relevant development requirements:
|
||||||
#. pip install -r tests/requirements.txt
|
#. pip install -r tests/requirements.txt
|
||||||
|
|
||||||
Once you have finished your changes, please provide test case(s), relevant documentation
|
Once you have finished your changes, please provide test case(s), relevant documentation
|
||||||
and update CHANGELOG.rst.
|
and update changelog.yml
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
|
|
||||||
|
@ -180,41 +224,20 @@ On Linux/Unix systems, please launch your tests like this::
|
||||||
|
|
||||||
$ make
|
$ make
|
||||||
|
|
||||||
On Windows systems, please issue this command::
|
On Windows, please issue this command::
|
||||||
|
|
||||||
> test.bat
|
> test.bat
|
||||||
|
|
||||||
How to update test environment and update documentation
|
|
||||||
---------------------------------------------------------
|
|
||||||
|
|
||||||
Additional steps are required:
|
Before you commit
|
||||||
|
------------------------------
|
||||||
|
|
||||||
#. pip install moban
|
Please run::
|
||||||
#. git clone https://github.com/moremoban/setupmobans.git # generic setup
|
|
||||||
#. git clone https://github.com/pyexcel/pyexcel-commons.git commons
|
|
||||||
#. make your changes in `.moban.d` directory, then issue command `moban`
|
|
||||||
|
|
||||||
What is pyexcel-commons
|
$ make format
|
||||||
---------------------------------
|
|
||||||
|
|
||||||
Many information that are shared across pyexcel projects, such as: this developer guide, license info, etc. are stored in `pyexcel-commons` project.
|
so as to beautify your code otherwise your build may fail your unit test.
|
||||||
|
|
||||||
What is .moban.d
|
|
||||||
---------------------------------
|
|
||||||
|
|
||||||
`.moban.d` stores the specific meta data for the library.
|
|
||||||
|
|
||||||
Acceptance criteria
|
|
||||||
-------------------
|
|
||||||
|
|
||||||
#. Has Test cases written
|
|
||||||
#. Has all code lines tested
|
|
||||||
#. Passes all Travis CI builds
|
|
||||||
#. Has fair amount of documentation if your change is complex
|
|
||||||
#. run 'make format' so as to confirm the pyexcel organisation's coding style
|
|
||||||
#. Please update CHANGELOG.rst
|
|
||||||
#. Please add yourself to CONTRIBUTORS.rst
|
|
||||||
#. Agree on NEW BSD License for your contribution
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,99 @@
|
||||||
name: pyexcel-io
|
name: pyexcel-io
|
||||||
organisation: pyexcel
|
organisation: pyexcel
|
||||||
releases:
|
releases:
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- "`#112`: Log Empty Row Warning instead 'print' "
|
||||||
|
version: 0.6.6
|
||||||
|
date: 31.1.2022
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- "`#109`: enable ods3 to have datetime"
|
||||||
|
version: 0.6.5
|
||||||
|
date: 08.10.2021
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- "`#102`: skip columns from imported excel sheet."
|
||||||
|
version: 0.6.4
|
||||||
|
date: 31.10.2020
|
||||||
|
- changes:
|
||||||
|
- action: fixed
|
||||||
|
details:
|
||||||
|
- "`#96`: regression: unknown file type shall trigger NoSupportingPluginFound"
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- "extra dependencies uses 0.6.0 based plugins"
|
||||||
|
version: 0.6.3
|
||||||
|
date: 12.10.2020
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- "`#94`: keep backward compatibility for pyexcel-xls 0.4.1"
|
||||||
|
version: 0.6.2
|
||||||
|
date: 7.10.2020
|
||||||
|
- changes:
|
||||||
|
- action: removed
|
||||||
|
details:
|
||||||
|
- 'python 3.6 lower versions are no longer supported'
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted.
|
||||||
|
please read 'Packaging with Pyinstaller' in the documentation.
|
||||||
|
- new query set reader plugin. pyexcel<=0.6.4 has used intrusive way of getting query set
|
||||||
|
source done. it is against the plugin interface.
|
||||||
|
- action: fixed
|
||||||
|
details:
|
||||||
|
- "`#74`: handle zip files which contain non-UTF-8 encoded files."
|
||||||
|
- action: added
|
||||||
|
details:
|
||||||
|
- "`#86`: allow trailing options, get_data(...keep_trailing_empty_cells=True)."
|
||||||
|
version: 0.6.1
|
||||||
|
date: 7.10.2020
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- '`#70`: when the given file is a root directory, the error shall read it is not a file'
|
||||||
|
version: 0.5.20
|
||||||
|
date: 17.7.2019
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- '`pyexcel#185`: handle stream conversion if file type(html) needs string content then bytes to handle'
|
||||||
|
version: 0.5.19
|
||||||
|
date: 14.7.2019
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- '`#69`: Force file type(force_file_type) on write'
|
||||||
|
version: 0.5.18
|
||||||
|
date: 12.06.2019
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- '`#68`: Raise IOError when the data file does not exist'
|
||||||
|
version: 0.5.17
|
||||||
|
date: 04.04.2019
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- '`#67`: fix conversion issue for long type on python 2.7 for ods'
|
||||||
|
version: 0.5.16
|
||||||
|
date: 19.03.2019
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- '`pyexcel-ods#33`: fix integer comparision error on i586'
|
||||||
|
version: 0.5.15
|
||||||
|
date: 16.03.2019
|
||||||
|
- changes:
|
||||||
|
- action: updated
|
||||||
|
details:
|
||||||
|
- '`#65`: add tests/__init__.py because python2.7 setup.py test needs it'
|
||||||
|
version: 0.5.14
|
||||||
|
date: 21.02.2019
|
||||||
- changes:
|
- changes:
|
||||||
- action: updated
|
- action: updated
|
||||||
details:
|
details:
|
||||||
|
@ -18,7 +111,7 @@ releases:
|
||||||
- action: updated
|
- action: updated
|
||||||
details:
|
details:
|
||||||
- '`#59`: Please use scan_plugins_regex, which lml 0.7 complains about'
|
- '`#59`: Please use scan_plugins_regex, which lml 0.7 complains about'
|
||||||
version: 0.5.10
|
version: 0.5.11
|
||||||
date: 3.12.2018
|
date: 3.12.2018
|
||||||
- changes:
|
- changes:
|
||||||
- action: added
|
- action: added
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
pyexcel_io.get_data
|
pyexcel\_io.get\_data
|
||||||
===================
|
=====================
|
||||||
|
|
||||||
.. currentmodule:: pyexcel_io
|
.. currentmodule:: pyexcel_io
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
pyexcel\_io.iget\_data
|
||||||
|
======================
|
||||||
|
|
||||||
|
.. currentmodule:: pyexcel_io
|
||||||
|
|
||||||
|
.. autofunction:: iget_data
|
|
@ -1,5 +1,5 @@
|
||||||
pyexcel_io.save_data
|
pyexcel\_io.save\_data
|
||||||
====================
|
======================
|
||||||
|
|
||||||
.. currentmodule:: pyexcel_io
|
.. currentmodule:: pyexcel_io
|
||||||
|
|
||||||
|
|
|
@ -2,9 +2,26 @@ Common parameters
|
||||||
================================================================================
|
================================================================================
|
||||||
|
|
||||||
|
|
||||||
|
'library' option is added
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
In order to have overlapping plugins co-exist, 'library' option is added to
|
||||||
|
get_data and save_data.
|
||||||
|
|
||||||
|
|
||||||
|
get_data only parameters
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
keep_trailing_empty_cells
|
||||||
|
********************************************************************************
|
||||||
|
|
||||||
|
default: False
|
||||||
|
|
||||||
|
If turned on, the return data will contain trailing empty cells.
|
||||||
|
|
||||||
|
|
||||||
auto_dectect_datetime
|
auto_dectect_datetime
|
||||||
--------------------------------------------------------------------------------
|
********************************************************************************
|
||||||
|
|
||||||
The datetime formats are:
|
The datetime formats are:
|
||||||
|
|
||||||
|
@ -14,11 +31,6 @@ The datetime formats are:
|
||||||
|
|
||||||
Any other datetime formats will be thrown as ValueError
|
Any other datetime formats will be thrown as ValueError
|
||||||
|
|
||||||
'library' option is added
|
|
||||||
--------------------------------------------------------------------------------
|
|
||||||
|
|
||||||
In order to have overlapping plugins co-exit, 'library' option is added to
|
|
||||||
get_data and save_data.
|
|
||||||
|
|
||||||
csv only parameters
|
csv only parameters
|
||||||
--------------------------------------------------------------------------------
|
--------------------------------------------------------------------------------
|
||||||
|
|
|
@ -4,13 +4,11 @@ DESCRIPTION = (
|
||||||
'format and to/from databases' +
|
'format and to/from databases' +
|
||||||
''
|
''
|
||||||
)
|
)
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
#
|
|
||||||
# Configuration file for the Sphinx documentation builder.
|
# Configuration file for the Sphinx documentation builder.
|
||||||
#
|
#
|
||||||
# This file does only contain a selection of the most common options. For a
|
# This file only contains a selection of the most common options. For a full
|
||||||
# full list see the documentation:
|
# list see the documentation:
|
||||||
# http://www.sphinx-doc.org/en/master/config
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||||
|
|
||||||
# -- Path setup --------------------------------------------------------------
|
# -- Path setup --------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -24,39 +22,24 @@ DESCRIPTION = (
|
||||||
|
|
||||||
# -- Project information -----------------------------------------------------
|
# -- Project information -----------------------------------------------------
|
||||||
|
|
||||||
project = u'pyexcel-io'
|
project = 'pyexcel-io'
|
||||||
copyright = u'2015-2018 Onni Software Ltd.'
|
copyright = '2015-2022 Onni Software Ltd.'
|
||||||
author = u'C.W.'
|
author = 'C.W.'
|
||||||
|
|
||||||
# The short X.Y version
|
# The short X.Y version
|
||||||
version = u'0.5.11'
|
version = '0.6.6'
|
||||||
# The full version, including alpha/beta/rc tags
|
# The full version, including alpha/beta/rc tags
|
||||||
release = u'0.5.11'
|
release = '0.6.6'
|
||||||
|
|
||||||
|
|
||||||
# -- General configuration ---------------------------------------------------
|
# -- General configuration ---------------------------------------------------
|
||||||
|
|
||||||
# If your documentation needs a minimal Sphinx version, state it here.
|
|
||||||
#
|
|
||||||
# needs_sphinx = '1.0'
|
|
||||||
|
|
||||||
# Add any Sphinx extension module names here, as strings. They can be
|
# Add any Sphinx extension module names here, as strings. They can be
|
||||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||||
# ones.
|
# ones.
|
||||||
extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode',]
|
extensions = [ 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', 'sphinx.ext.autodoc',]
|
||||||
|
|
||||||
# Add any paths that contain templates here, relative to this directory.
|
# Add any paths that contain templates here, relative to this directory.
|
||||||
templates_path = ['_templates']
|
templates_path = ['_templates']
|
||||||
|
|
||||||
# The suffix(es) of source filenames.
|
|
||||||
# You can specify multiple suffix as a list of string:
|
|
||||||
#
|
|
||||||
# source_suffix = ['.rst', '.md']
|
|
||||||
source_suffix = '.rst'
|
|
||||||
|
|
||||||
# The master toctree document.
|
|
||||||
master_doc = 'index'
|
|
||||||
|
|
||||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||||
# for a list of supported languages.
|
# for a list of supported languages.
|
||||||
#
|
#
|
||||||
|
@ -69,9 +52,6 @@ language = 'en'
|
||||||
# This pattern also affects html_static_path and html_extra_path.
|
# This pattern also affects html_static_path and html_extra_path.
|
||||||
exclude_patterns = []
|
exclude_patterns = []
|
||||||
|
|
||||||
# The name of the Pygments (syntax highlighting) style to use.
|
|
||||||
pygments_style = None
|
|
||||||
|
|
||||||
|
|
||||||
# -- Options for HTML output -------------------------------------------------
|
# -- Options for HTML output -------------------------------------------------
|
||||||
|
|
||||||
|
@ -80,107 +60,16 @@ pygments_style = None
|
||||||
#
|
#
|
||||||
html_theme = 'alabaster'
|
html_theme = 'alabaster'
|
||||||
|
|
||||||
# Theme options are theme-specific and customize the look and feel of a theme
|
|
||||||
# further. For a list of options available for each theme, see the
|
|
||||||
# documentation.
|
|
||||||
#
|
|
||||||
# html_theme_options = {}
|
|
||||||
|
|
||||||
# Add any paths that contain custom static files (such as style sheets) here,
|
# Add any paths that contain custom static files (such as style sheets) here,
|
||||||
# relative to this directory. They are copied after the builtin static files,
|
# relative to this directory. They are copied after the builtin static files,
|
||||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||||
html_static_path = ['_static']
|
html_static_path = ['_static']
|
||||||
|
|
||||||
# Custom sidebar templates, must be a dictionary that maps document names
|
|
||||||
# to template names.
|
|
||||||
#
|
|
||||||
# The default sidebars (for documents that don't match any pattern) are
|
|
||||||
# defined by theme itself. Builtin themes are using these templates by
|
|
||||||
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
|
|
||||||
# 'searchbox.html']``.
|
|
||||||
#
|
|
||||||
# html_sidebars = {}
|
|
||||||
|
|
||||||
|
|
||||||
# -- Options for HTMLHelp output ---------------------------------------------
|
|
||||||
|
|
||||||
# Output file base name for HTML help builder.
|
|
||||||
htmlhelp_basename = 'pyexcel-iodoc'
|
|
||||||
|
|
||||||
|
|
||||||
# -- Options for LaTeX output ------------------------------------------------
|
|
||||||
|
|
||||||
latex_elements = {
|
|
||||||
# The paper size ('letterpaper' or 'a4paper').
|
|
||||||
#
|
|
||||||
# 'papersize': 'letterpaper',
|
|
||||||
|
|
||||||
# The font size ('10pt', '11pt' or '12pt').
|
|
||||||
#
|
|
||||||
# 'pointsize': '10pt',
|
|
||||||
|
|
||||||
# Additional stuff for the LaTeX preamble.
|
|
||||||
#
|
|
||||||
# 'preamble': '',
|
|
||||||
|
|
||||||
# Latex figure (float) alignment
|
|
||||||
#
|
|
||||||
# 'figure_align': 'htbp',
|
|
||||||
}
|
|
||||||
|
|
||||||
# Grouping the document tree into LaTeX files. List of tuples
|
|
||||||
# (source start file, target name, title,
|
|
||||||
# author, documentclass [howto, manual, or own class]).
|
|
||||||
latex_documents = [
|
|
||||||
(master_doc, 'pyexcel-io.tex', u'pyexcel-io Documentation',
|
|
||||||
u'Onni Software Ltd.', 'manual'),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
# -- Options for manual page output ------------------------------------------
|
|
||||||
|
|
||||||
# One entry per manual page. List of tuples
|
|
||||||
# (source start file, name, description, authors, manual section).
|
|
||||||
man_pages = [
|
|
||||||
(master_doc, 'pyexcel-io', u'pyexcel-io Documentation',
|
|
||||||
[author], 1)
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
# -- Options for Texinfo output ----------------------------------------------
|
|
||||||
|
|
||||||
# Grouping the document tree into Texinfo files. List of tuples
|
|
||||||
# (source start file, target name, title, author,
|
|
||||||
# dir menu entry, description, category)
|
|
||||||
texinfo_documents = [
|
|
||||||
(master_doc, 'pyexcel-io', u'pyexcel-io Documentation',
|
|
||||||
author, 'pyexcel-io', 'One line description of project.',
|
|
||||||
'Miscellaneous'),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
# -- Options for Epub output -------------------------------------------------
|
|
||||||
|
|
||||||
# Bibliographic Dublin Core info.
|
|
||||||
epub_title = project
|
|
||||||
|
|
||||||
# The unique identifier of the text. This can be a ISBN number
|
|
||||||
# or the project homepage.
|
|
||||||
#
|
|
||||||
# epub_identifier = ''
|
|
||||||
|
|
||||||
# A unique identification for the text.
|
|
||||||
#
|
|
||||||
# epub_uid = ''
|
|
||||||
|
|
||||||
# A list of files that should not be packed into the epub file.
|
|
||||||
epub_exclude_files = ['search.html']
|
|
||||||
|
|
||||||
# -- Extension configuration -------------------------------------------------
|
# -- Extension configuration -------------------------------------------------
|
||||||
# -- Options for intersphinx extension ---------------------------------------
|
# -- Options for intersphinx extension ---------------------------------------
|
||||||
|
|
||||||
# Example configuration for intersphinx: refer to the Python standard library.
|
# Example configuration for intersphinx: refer to the Python standard library.
|
||||||
intersphinx_mapping = {'https://docs.python.org/': None}
|
intersphinx_mapping = {'https://docs.python.org/3/': None}
|
||||||
# TODO: html_theme not configurable upstream
|
# TODO: html_theme not configurable upstream
|
||||||
html_theme = 'default'
|
html_theme = 'default'
|
||||||
|
|
||||||
|
@ -201,3 +90,4 @@ texinfo_documents = [
|
||||||
intersphinx_mapping.update({
|
intersphinx_mapping.update({
|
||||||
'pyexcel': ('http://pyexcel.readthedocs.io/en/latest/', None),
|
'pyexcel': ('http://pyexcel.readthedocs.io/en/latest/', None),
|
||||||
})
|
})
|
||||||
|
master_doc = "index"
|
||||||
|
|
|
@ -74,6 +74,8 @@ Continue from previous example::
|
||||||
:hide:
|
:hide:
|
||||||
|
|
||||||
>>> import os
|
>>> import os
|
||||||
|
>>> if os.path.exists("your_file.csv"):
|
||||||
|
... os.unlink("your_file.csv")
|
||||||
>>> os.unlink("your_file__Sheet 1__0.csv")
|
>>> os.unlink("your_file__Sheet 1__0.csv")
|
||||||
>>> os.unlink("your_file__Sheet 2__1.csv")
|
>>> os.unlink("your_file__Sheet 2__1.csv")
|
||||||
|
|
||||||
|
|
|
@ -1,14 +1,160 @@
|
||||||
Working with xls, xlsx, and ods formats
|
Extend pyexcel-io for other excel or tabular formats
|
||||||
================================================================================
|
================================================================================
|
||||||
|
|
||||||
|
You are welcome to extend pyexcel-io to read and write more tabular formats.
|
||||||
|
No. 1 rule, your plugin must have a prefix 'pyexcel_' in its module path.
|
||||||
|
For example, `pyexcel-xls` has 'pyexcel_xls' as its module path. Otherwise,
|
||||||
|
pyexcel-io will not load your plugin.
|
||||||
|
|
||||||
|
On github, you will find two examples in `examples` folder. This section
|
||||||
|
explains its implementations to help you write yours.
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
|
|
||||||
No longer, you will need to do explicit imports for pyexcel-io extensions.
|
No longer, you will need to do explicit imports for pyexcel-io extensions.
|
||||||
Instead, you install them and manage them via pip.
|
Instead, you install them and manage them via pip.
|
||||||
|
|
||||||
Work with physical file
|
Simple Reader for a yaml file
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
Suppose we have a yaml file, containing a dictionary where the values are
|
||||||
|
two dimensional array. The task is to write a reader plugin to pyexcel-io so that
|
||||||
|
we can use get_data() to read yaml file out.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/test.yaml
|
||||||
|
:language: yaml
|
||||||
|
|
||||||
|
**Implement IReader**
|
||||||
|
|
||||||
|
First, let's implement reader interface:
|
||||||
|
|
||||||
|
1. `content_array` attribute, is expected to be a list of `NamedContent`
|
||||||
|
2. `read_sheet` function, read sheet content by its index.
|
||||||
|
3. `close` function, to clean up any file handle
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/custom_yaml_reader.py
|
||||||
|
:language: python
|
||||||
|
:lines: 19-33
|
||||||
|
|
||||||
|
**Implement ISheet**
|
||||||
|
|
||||||
|
`YourSingleSheet` makes this simple task complex in order to show case its inner
|
||||||
|
workings. Two abstract functions require implementation:
|
||||||
|
|
||||||
|
1. `row_iterator`: should return a row: either content arary or content index as long as
|
||||||
|
`column_iterator` can use it to return the cell value.
|
||||||
|
|
||||||
|
2. `column_iterator`: should iterate cell value from the given row.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/custom_yaml_reader.py
|
||||||
|
:language: python
|
||||||
|
:lines: 8-16
|
||||||
|
|
||||||
|
|
||||||
|
**Plug in pyexcel-io**
|
||||||
|
|
||||||
|
Last thing is to register with pyexcel-io about your new reader. `relative_plugin_class_path`
|
||||||
|
meant reference from current module, how to refer to `YourReader`. `locations` meant
|
||||||
|
the physical presence of the data source: "file", "memory" or "content". "file" means
|
||||||
|
files on physical disk. "memory" means a file stream. "content" means a string buffer.
|
||||||
|
`stream_type` meant the type of the stream: binary for BytesIO and text for StringIO.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/custom_yaml_reader.py
|
||||||
|
:language: python
|
||||||
|
:lines: 36-41
|
||||||
|
|
||||||
|
Usually, this registration code was placed in __init__.py file at the top level of your
|
||||||
|
extension source tree. You can take a look at any pyexcel plugins for reference.
|
||||||
|
|
||||||
|
**Test your reader**
|
||||||
|
|
||||||
|
Let's run the following code and see if it works.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/custom_yaml_reader.py
|
||||||
|
:language: python
|
||||||
|
:lines: 43-45
|
||||||
|
|
||||||
|
|
||||||
|
You would see these in standard output:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
$ python custom_yaml_reader.py
|
||||||
|
OrderedDict([('sheet 1', [[1, 2, 3], [2, 3, 4]]), ('sheet 2', [['A', 'B', 'C']])])
|
||||||
|
|
||||||
|
A writer to write content in yaml
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
Now for the writer, let's write a pyexcel-io writer that write a dictionary of
|
||||||
|
two dimentaional arrays back into a yaml file seen above.
|
||||||
|
|
||||||
|
**Implement IWriter**
|
||||||
|
|
||||||
|
Two abstract functions are required:
|
||||||
|
|
||||||
|
1. `create_sheet` creates a native sheet by sheet name, that understands how to code up the native sheet. Interestingly, it returns your sheet.
|
||||||
|
2. `close` function closes file handle if any.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/custom_yaml_writer.py
|
||||||
|
:language: python
|
||||||
|
:lines: 18-30
|
||||||
|
|
||||||
|
**Implement ISheetWriter**
|
||||||
|
|
||||||
|
It is imagined that you will have your own sheet writer. You simply need to figure
|
||||||
|
out how to write a row. Row by row write action was already written by `ISheetWrier`.
|
||||||
|
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/custom_yaml_writer.py
|
||||||
|
:language: python
|
||||||
|
:lines: 7-14
|
||||||
|
|
||||||
|
**Plug in pyexcel-io**
|
||||||
|
|
||||||
|
Like the reader plugin, we register a writer.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/custom_yaml_writer.py
|
||||||
|
:language: python
|
||||||
|
:lines: 33-38
|
||||||
|
|
||||||
|
**Test It**
|
||||||
|
|
||||||
|
Let's run the following code and please examine `mytest.yaml` yourself.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../examples/custom_yaml_writer.py
|
||||||
|
:language: python
|
||||||
|
:lines: 40-46
|
||||||
|
|
||||||
|
And you shall find a file named 'mytest.yaml':
|
||||||
|
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
$ cat mytest.yaml
|
||||||
|
sheet 1:
|
||||||
|
- - 1
|
||||||
|
- 3
|
||||||
|
- 4
|
||||||
|
- - 2
|
||||||
|
- 4
|
||||||
|
- 9
|
||||||
|
sheet 2:
|
||||||
|
- - B
|
||||||
|
- C
|
||||||
|
- D
|
||||||
|
|
||||||
|
|
||||||
|
Other pyexcel-io plugins
|
||||||
-----------------------------------------------------------------------------
|
-----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
Get xls support
|
||||||
|
|
||||||
|
.. code-block::
|
||||||
|
|
||||||
|
|
||||||
|
$ pip install pyexcel-xls
|
||||||
|
|
||||||
|
|
||||||
Here's what is needed::
|
Here's what is needed::
|
||||||
|
|
||||||
>>> from pyexcel_io import save_data
|
>>> from pyexcel_io import save_data
|
||||||
|
@ -23,27 +169,6 @@ And you can also get the data back::
|
||||||
[[1, 2, 3]]
|
[[1, 2, 3]]
|
||||||
|
|
||||||
|
|
||||||
Work with memory file
|
|
||||||
-----------------------------------------------------------------------------
|
|
||||||
|
|
||||||
Here is the sample code to work with memory file::
|
|
||||||
|
|
||||||
>>> from pyexcel_io.manager import get_io
|
|
||||||
>>> io = get_io("xls")
|
|
||||||
>>> data = [[1,2,3]]
|
|
||||||
>>> save_data(io, data, "xls")
|
|
||||||
|
|
||||||
The difference is that you have mention file type if you use :meth:`pyexcel_io.save_data`
|
|
||||||
|
|
||||||
And you can also get the data back::
|
|
||||||
|
|
||||||
>>> data = get_data(io, "xls")
|
|
||||||
>>> data['pyexcel_sheet1']
|
|
||||||
[[1, 2, 3]]
|
|
||||||
|
|
||||||
The same applies to :meth:`pyexcel_io.get_data`.
|
|
||||||
|
|
||||||
|
|
||||||
Other formats
|
Other formats
|
||||||
-----------------------------------------------------------------------------
|
-----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
|
@ -64,48 +64,47 @@ For individual excel file formats, please install them as you wish:
|
||||||
|
|
||||||
.. table:: A list of file formats supported by external plugins
|
.. table:: A list of file formats supported by external plugins
|
||||||
|
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
Package name Supported file formats Dependencies Python versions
|
Package name Supported file formats Dependencies
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
`pyexcel-io`_ csv, csvz [#f1]_, tsv, 2.6, 2.7, 3.3,
|
`pyexcel-io`_ csv, csvz [#f1]_, tsv,
|
||||||
tsvz [#f2]_ 3.4, 3.5, 3.6
|
tsvz [#f2]_
|
||||||
pypy
|
`pyexcel-xls`_ xls, xlsx(read only), `xlrd`_,
|
||||||
`pyexcel-xls`_ xls, xlsx(read only), `xlrd`_, same as above
|
|
||||||
xlsm(read only) `xlwt`_
|
xlsm(read only) `xlwt`_
|
||||||
`pyexcel-xlsx`_ xlsx `openpyxl`_ same as above
|
`pyexcel-xlsx`_ xlsx `openpyxl`_
|
||||||
`pyexcel-ods3`_ ods `pyexcel-ezodf`_, 2.6, 2.7, 3.3, 3.4
|
`pyexcel-ods3`_ ods `pyexcel-ezodf`_,
|
||||||
lxml 3.5, 3.6
|
lxml
|
||||||
`pyexcel-ods`_ ods `odfpy`_ same as above
|
`pyexcel-ods`_ ods `odfpy`_
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
|
|
||||||
.. table:: Dedicated file reader and writers
|
.. table:: Dedicated file reader and writers
|
||||||
|
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
Package name Supported file formats Dependencies Python versions
|
Package name Supported file formats Dependencies
|
||||||
======================== ======================= ================= ==================
|
======================== ======================= =================
|
||||||
`pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_ Python 2 and 3
|
`pyexcel-xlsxw`_ xlsx(write only) `XlsxWriter`_
|
||||||
`pyexcel-xlsxr`_ xlsx(read only) lxml same as above
|
`pyexcel-libxlsxw`_ xlsx(write only) `libxlsxwriter`_
|
||||||
`pyexcel-odsr`_ read only for ods, fods lxml same as above
|
`pyexcel-xlsxr`_ xlsx(read only) lxml
|
||||||
`pyexcel-htmlr`_ html(read only) lxml,html5lib same as above
|
`pyexcel-xlsbr`_ xlsb(read only) pyxlsb
|
||||||
======================== ======================= ================= ==================
|
`pyexcel-odsr`_ read only for ods, fods lxml
|
||||||
|
`pyexcel-odsw`_ write only for ods loxun
|
||||||
|
`pyexcel-htmlr`_ html(read only) lxml,html5lib
|
||||||
|
`pyexcel-pdfr`_ pdf(read only) camelot
|
||||||
|
======================== ======================= =================
|
||||||
|
|
||||||
|
|
||||||
.. _pyexcel-io: https://github.com/pyexcel/pyexcel-io
|
Plugin shopping guide
|
||||||
.. _pyexcel-xls: https://github.com/pyexcel/pyexcel-xls
|
------------------------
|
||||||
.. _pyexcel-xlsx: https://github.com/pyexcel/pyexcel-xlsx
|
|
||||||
.. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods
|
|
||||||
.. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3
|
|
||||||
.. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr
|
|
||||||
.. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw
|
|
||||||
.. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr
|
|
||||||
.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr
|
|
||||||
|
|
||||||
.. _xlrd: https://github.com/python-excel/xlrd
|
Since 2020, all pyexcel-io plugins have dropped the support for python versions
|
||||||
.. _xlwt: https://github.com/python-excel/xlwt
|
which are lower than 3.6. If you want to use any of those Python versions, please use pyexcel-io
|
||||||
.. _openpyxl: https://bitbucket.org/openpyxl/openpyxl
|
and its plugins versions that are lower than 0.6.0.
|
||||||
.. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter
|
|
||||||
.. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf
|
|
||||||
.. _odfpy: https://github.com/eea/odfpy
|
Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of
|
||||||
|
xml files
|
||||||
|
|
||||||
|
The dedicated readers for excel files can stream read
|
||||||
|
|
||||||
|
|
||||||
In order to manage the list of plugins installed, you need to use pip to add or remove
|
In order to manage the list of plugins installed, you need to use pip to add or remove
|
||||||
|
@ -115,6 +114,32 @@ in your environment, you need to tell pyexcel which plugin to use per function c
|
||||||
For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr.
|
For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr.
|
||||||
You need to append get_array(..., library='pyexcel-odsr').
|
You need to append get_array(..., library='pyexcel-odsr').
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
.. _pyexcel-io: https://github.com/pyexcel/pyexcel-io
|
||||||
|
.. _pyexcel-xls: https://github.com/pyexcel/pyexcel-xls
|
||||||
|
.. _pyexcel-xlsx: https://github.com/pyexcel/pyexcel-xlsx
|
||||||
|
.. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods
|
||||||
|
.. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3
|
||||||
|
.. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr
|
||||||
|
.. _pyexcel-odsw: https://github.com/pyexcel/pyexcel-odsw
|
||||||
|
.. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr
|
||||||
|
|
||||||
|
.. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw
|
||||||
|
.. _pyexcel-libxlsxw: https://github.com/pyexcel/pyexcel-libxlsxw
|
||||||
|
.. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr
|
||||||
|
.. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr
|
||||||
|
.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr
|
||||||
|
|
||||||
|
.. _xlrd: https://github.com/python-excel/xlrd
|
||||||
|
.. _xlwt: https://github.com/python-excel/xlwt
|
||||||
|
.. _openpyxl: https://bitbucket.org/openpyxl/openpyxl
|
||||||
|
.. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter
|
||||||
|
.. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf
|
||||||
|
.. _odfpy: https://github.com/eea/odfpy
|
||||||
|
.. _libxlsxwriter: http://libxlsxwriter.github.io/getting_started.html
|
||||||
|
|
||||||
|
|
||||||
.. rubric:: Footnotes
|
.. rubric:: Footnotes
|
||||||
|
|
||||||
.. [#f1] zipped csv file
|
.. [#f1] zipped csv file
|
||||||
|
@ -138,6 +163,7 @@ get_data(.., library='pyexcel-ods')
|
||||||
============= ======= ======== ======= ======== ======== ========
|
============= ======= ======== ======= ======== ======== ========
|
||||||
`pyexcel-io`_ `xls`_ `xlsx`_ `ods`_ `ods3`_ `odsr`_ `xlsxw`_
|
`pyexcel-io`_ `xls`_ `xlsx`_ `ods`_ `ods3`_ `odsr`_ `xlsxw`_
|
||||||
============= ======= ======== ======= ======== ======== ========
|
============= ======= ======== ======= ======== ======== ========
|
||||||
|
0.6.0+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+
|
||||||
0.5.10+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+
|
0.5.10+ 0.5.0+ 0.5.0+ 0.5.4 0.5.3 0.5.0+ 0.5.0+
|
||||||
0.5.1+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+
|
0.5.1+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+ 0.5.0+
|
||||||
0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x
|
0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x 0.4.x
|
||||||
|
@ -188,6 +214,7 @@ API
|
||||||
.. autosummary::
|
.. autosummary::
|
||||||
:toctree: api/
|
:toctree: api/
|
||||||
|
|
||||||
|
iget_data
|
||||||
get_data
|
get_data
|
||||||
save_data
|
save_data
|
||||||
|
|
||||||
|
|
|
@ -60,3 +60,9 @@ Obvious, you could do both at the same time:
|
||||||
|
|
||||||
The pagination support is available across all pyexcel-io plugins.
|
The pagination support is available across all pyexcel-io plugins.
|
||||||
|
|
||||||
|
.. testcode::
|
||||||
|
:hide:
|
||||||
|
|
||||||
|
>>> import os
|
||||||
|
>>> if os.path.exists("your_file.csv"):
|
||||||
|
... os.unlink("your_file.csv")
|
||||||
|
|
|
@ -153,7 +153,7 @@ Here is an example to write a sentence of "Shui Dial Getou"[#f2] into a csv file
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
>>> content = [[u'人有悲歡離合', u'月有陰晴圓缺']]
|
>>> content = [['löyly', 'löyly']]
|
||||||
>>> test_file = "test-utf8-BOM.csv"
|
>>> test_file = "test-utf8-BOM.csv"
|
||||||
>>> save_data(test_file, content, encoding="utf-8-sig", lineterminator="\n")
|
>>> save_data(test_file, content, encoding="utf-8-sig", lineterminator="\n")
|
||||||
|
|
||||||
|
@ -172,4 +172,5 @@ When you read it back you will have to specify encoding too.
|
||||||
|
|
||||||
>>> import os
|
>>> import os
|
||||||
>>> os.unlink("your_file.csv")
|
>>> os.unlink("your_file.csv")
|
||||||
|
>>> os.unlink("test-utf16-encoding.csv")
|
||||||
>>> os.unlink(test_file)
|
>>> os.unlink(test_file)
|
||||||
|
|
|
@ -1,6 +1,27 @@
|
||||||
Packaging with PyInstaller
|
Packaging with PyInstaller
|
||||||
================================================================================
|
================================================================================
|
||||||
|
|
||||||
|
With pyexcel-io v0.6.0, the way to package it has been changed because
|
||||||
|
plugin interface update.
|
||||||
|
|
||||||
|
Built-in plugins for pyexcel-io
|
||||||
|
---------------------------------
|
||||||
|
|
||||||
|
In order to package every built-in plugins of pyexcel-io, you need to specify::
|
||||||
|
|
||||||
|
--hidden-import pyexcel_io.readers.csv_in_file
|
||||||
|
--hidden-import pyexcel_io.readers.csv_in_memory
|
||||||
|
--hidden-import pyexcel_io.readers.csv_content
|
||||||
|
--hidden-import pyexcel_io.readers.csvz
|
||||||
|
--hidden-import pyexcel_io.writers.csv_in_file
|
||||||
|
--hidden-import pyexcel_io.writers.csv_in_memory
|
||||||
|
--hidden-import pyexcel_io.writers.csvz_writer
|
||||||
|
--hidden-import pyexcel_io.database.importers.django
|
||||||
|
--hidden-import pyexcel_io.database.importers.sqlalchemy
|
||||||
|
--hidden-import pyexcel_io.database.exporters.django
|
||||||
|
--hidden-import pyexcel_io.database.exporters.sqlalchemy
|
||||||
|
|
||||||
|
|
||||||
With pyexcel-io v0.4.0, the way to package it has been changed because it
|
With pyexcel-io v0.4.0, the way to package it has been changed because it
|
||||||
uses lml for all plugins.
|
uses lml for all plugins.
|
||||||
|
|
||||||
|
@ -14,9 +35,9 @@ In order to package every built-in plugins of pyexcel-io, you need to specify::
|
||||||
--hidden-import pyexcel_io.readers.tsv
|
--hidden-import pyexcel_io.readers.tsv
|
||||||
--hidden-import pyexcel_io.readers.tsvz
|
--hidden-import pyexcel_io.readers.tsvz
|
||||||
--hidden-import pyexcel_io.writers.csvw
|
--hidden-import pyexcel_io.writers.csvw
|
||||||
--hidden-import pyexcel_io.readers.csvz
|
--hidden-import pyexcel_io.writers.csvz
|
||||||
--hidden-import pyexcel_io.readers.tsv
|
--hidden-import pyexcel_io.writers.tsv
|
||||||
--hidden-import pyexcel_io.readers.tsvz
|
--hidden-import pyexcel_io.writers.tsvz
|
||||||
--hidden-import pyexcel_io.database.importers.django
|
--hidden-import pyexcel_io.database.importers.django
|
||||||
--hidden-import pyexcel_io.database.importers.sqlalchemy
|
--hidden-import pyexcel_io.database.importers.sqlalchemy
|
||||||
--hidden-import pyexcel_io.database.exporters.django
|
--hidden-import pyexcel_io.database.exporters.django
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
Rendering(Formatting) the data
|
Rendering(Formatting) the data
|
||||||
|
|
||||||
================================================================================
|
================================================================================
|
||||||
|
|
||||||
You might want to do custom rendering on your data obtained. `row_renderer` was
|
You might want to do custom rendering on your data obtained. `row_renderer` was
|
||||||
|
@ -46,3 +45,10 @@ And you may want use row_renderer to customize it to string:
|
||||||
>>> data = get_data("your_file.csv", row_renderer=my_renderer)
|
>>> data = get_data("your_file.csv", row_renderer=my_renderer)
|
||||||
>>> data['your_file.csv']
|
>>> data['your_file.csv']
|
||||||
[['1', '21', '31'], ['2', '22', '32'], ['3', '23', '33']]
|
[['1', '21', '31'], ['2', '22', '32'], ['3', '23', '33']]
|
||||||
|
|
||||||
|
.. testcode::
|
||||||
|
:hide:
|
||||||
|
|
||||||
|
>>> import os
|
||||||
|
>>> if os.path.exists("your_file.csv"):
|
||||||
|
... os.unlink("your_file.csv")
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
import yaml
|
||||||
|
from pyexcel_io import get_data
|
||||||
|
from pyexcel_io.sheet import NamedContent
|
||||||
|
from pyexcel_io.plugins import IOPluginInfoChainV2
|
||||||
|
from pyexcel_io.plugin_api import ISheet, IReader
|
||||||
|
|
||||||
|
|
||||||
|
class YourSingleSheet(ISheet):
|
||||||
|
def __init__(self, your_native_sheet):
|
||||||
|
self.two_dimensional_array = your_native_sheet
|
||||||
|
|
||||||
|
def row_iterator(self):
|
||||||
|
yield from self.two_dimensional_array
|
||||||
|
|
||||||
|
def column_iterator(self, row):
|
||||||
|
yield from row
|
||||||
|
|
||||||
|
|
||||||
|
class YourReader(IReader):
|
||||||
|
def __init__(self, file_name, file_type, **keywords):
|
||||||
|
self.file_handle = open(file_name, "r")
|
||||||
|
self.native_book = yaml.load(self.file_handle)
|
||||||
|
self.content_array = [
|
||||||
|
NamedContent(key, values)
|
||||||
|
for key, values in self.native_book.items()
|
||||||
|
]
|
||||||
|
|
||||||
|
def read_sheet(self, sheet_index):
|
||||||
|
two_dimensional_array = self.content_array[sheet_index].payload
|
||||||
|
return YourSingleSheet(two_dimensional_array)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.file_handle.close()
|
||||||
|
|
||||||
|
|
||||||
|
IOPluginInfoChainV2(__name__).add_a_reader(
|
||||||
|
relative_plugin_class_path="YourReader",
|
||||||
|
locations=["file"],
|
||||||
|
file_types=["yaml"],
|
||||||
|
stream_type="text",
|
||||||
|
)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
data = get_data("test.yaml")
|
||||||
|
print(data)
|
|
@ -0,0 +1,46 @@
|
||||||
|
import yaml
|
||||||
|
from pyexcel_io import save_data
|
||||||
|
from pyexcel_io.plugins import IOPluginInfoChainV2
|
||||||
|
from pyexcel_io.plugin_api import IWriter, ISheetWriter
|
||||||
|
|
||||||
|
|
||||||
|
class MySheetWriter(ISheetWriter):
|
||||||
|
def __init__(self, sheet_reference):
|
||||||
|
self.native_sheet = sheet_reference
|
||||||
|
|
||||||
|
def write_row(self, data_row):
|
||||||
|
self.native_sheet.append(data_row)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class MyWriter(IWriter):
|
||||||
|
def __init__(self, file_name, file_type, **keywords):
|
||||||
|
self.file_name = file_name
|
||||||
|
self.content = {}
|
||||||
|
|
||||||
|
def create_sheet(self, name):
|
||||||
|
array = []
|
||||||
|
self.content[name] = array
|
||||||
|
return MySheetWriter(array)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
with open(self.file_name, "w") as f:
|
||||||
|
f.write(yaml.dump(self.content, default_flow_style=False))
|
||||||
|
|
||||||
|
|
||||||
|
IOPluginInfoChainV2(__name__).add_a_writer(
|
||||||
|
relative_plugin_class_path="MyWriter",
|
||||||
|
locations=["file"],
|
||||||
|
file_types=["yaml"],
|
||||||
|
stream_type="text",
|
||||||
|
)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
data_dict = {
|
||||||
|
"sheet 1": [[1, 3, 4], [2, 4, 9]],
|
||||||
|
"sheet 2": [["B", "C", "D"]],
|
||||||
|
}
|
||||||
|
|
||||||
|
save_data("mytest.yaml", data_dict)
|
|
@ -0,0 +1,11 @@
|
||||||
|
sheet 1:
|
||||||
|
- - 1
|
||||||
|
- 2
|
||||||
|
- 3
|
||||||
|
- - 2
|
||||||
|
- 3
|
||||||
|
- 4
|
||||||
|
sheet 2:
|
||||||
|
- - A
|
||||||
|
- B
|
||||||
|
- C
|
|
@ -0,0 +1,3 @@
|
||||||
|
isort $(find pyexcel_io -name "*.py"|xargs echo) $(find tests -name "*.py"|xargs echo)
|
||||||
|
black -l 79 pyexcel_io
|
||||||
|
black -l 79 tests
|
|
@ -0,0 +1,2 @@
|
||||||
|
pip install flake8
|
||||||
|
flake8 --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long . && python setup.py checkdocs
|
|
@ -1,19 +1,27 @@
|
||||||
overrides: "pyexcel.yaml"
|
overrides: "pyexcel.yaml"
|
||||||
name: "pyexcel-io"
|
project: "pyexcel-io"
|
||||||
|
name: pyexcel-io
|
||||||
nick_name: io
|
nick_name: io
|
||||||
version: 0.5.13
|
version: 0.6.6
|
||||||
current_version: 0.5.13
|
current_version: 0.6.6
|
||||||
release: 0.5.13
|
release: 0.6.6
|
||||||
|
copyright_year: 2015-2022
|
||||||
|
moban_command: false
|
||||||
|
is_on_conda: true
|
||||||
dependencies:
|
dependencies:
|
||||||
- ordereddict;python_version<"2.7"
|
|
||||||
- lml>=0.0.4
|
- lml>=0.0.4
|
||||||
|
test_dependencies:
|
||||||
|
- pyexcel
|
||||||
|
- pyexcel-xls==0.5.9
|
||||||
|
- SQLAlchemy
|
||||||
|
- pyexcel-xlsxw
|
||||||
extra_dependencies:
|
extra_dependencies:
|
||||||
- xls:
|
- xls:
|
||||||
- pyexcel-xls>=0.5.0
|
- pyexcel-xls>=0.6.0
|
||||||
- xlsx:
|
- xlsx:
|
||||||
- pyexcel-xlsx>=0.5.0
|
- pyexcel-xlsx>=0.6.0
|
||||||
- ods:
|
- ods:
|
||||||
- pyexcel-ods3>=0.5.0
|
- pyexcel-ods3>=0.6.0
|
||||||
keywords:
|
keywords:
|
||||||
- API
|
- API
|
||||||
- tsv
|
- tsv
|
||||||
|
@ -22,4 +30,12 @@ keywords:
|
||||||
- csvz
|
- csvz
|
||||||
- django
|
- django
|
||||||
- sqlalchemy
|
- sqlalchemy
|
||||||
|
sphinx_extensions:
|
||||||
|
- sphinx.ext.autosummary
|
||||||
|
- sphinx.ext.doctest
|
||||||
|
- sphinx.ext.intersphinx
|
||||||
|
- sphinx.ext.viewcode
|
||||||
|
- sphinx.ext.autodoc
|
||||||
description: A python library to read and write structured data in csv, zipped csv format and to/from databases
|
description: A python library to read and write structured data in csv, zipped csv format and to/from databases
|
||||||
|
python_requires: ">=3.6"
|
||||||
|
min_python_version: "3.6"
|
||||||
|
|
|
@ -4,17 +4,18 @@
|
||||||
|
|
||||||
Uniform interface for reading/writing different excel file formats
|
Uniform interface for reading/writing different excel file formats
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
import pyexcel_io.plugins as plugins
|
||||||
|
|
||||||
|
from .io import get_data, iget_data, save_data # noqa
|
||||||
from ._compact import NullHandler
|
from ._compact import NullHandler
|
||||||
|
|
||||||
logging.getLogger(__name__).addHandler(NullHandler()) # noqa
|
logging.getLogger(__name__).addHandler(NullHandler()) # noqa
|
||||||
|
|
||||||
from .io import get_data, iget_data, save_data # noqa
|
|
||||||
import pyexcel_io.plugins as plugins
|
|
||||||
|
|
||||||
|
|
||||||
BLACK_LIST = [__name__, "pyexcel_webio", "pyexcel_text"]
|
BLACK_LIST = [__name__, "pyexcel_webio", "pyexcel_text"]
|
||||||
WHITE_LIST = [
|
WHITE_LIST = [
|
||||||
|
@ -25,7 +26,5 @@ WHITE_LIST = [
|
||||||
PREFIX_PATTERN = "^pyexcel_.*$"
|
PREFIX_PATTERN = "^pyexcel_.*$"
|
||||||
|
|
||||||
plugins.load_plugins(
|
plugins.load_plugins(
|
||||||
PREFIX_PATTERN,
|
PREFIX_PATTERN, __path__, BLACK_LIST, WHITE_LIST # noqa: F821
|
||||||
__path__, # noqa: F821
|
)
|
||||||
BLACK_LIST,
|
|
||||||
WHITE_LIST)
|
|
||||||
|
|
|
@ -4,29 +4,13 @@
|
||||||
|
|
||||||
Compatibles
|
Compatibles
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
# flake8: noqa
|
|
||||||
# pylint: disable=import-error
|
|
||||||
# pylint: disable=invalid-name
|
|
||||||
# pylint: disable=too-few-public-methods
|
|
||||||
# pylint: disable=ungrouped-imports
|
|
||||||
# pylint: disable=redefined-variable-type
|
|
||||||
import sys
|
import sys
|
||||||
import types
|
|
||||||
import logging
|
import logging
|
||||||
|
from io import BytesIO, StringIO # noqa: F401
|
||||||
PY2 = sys.version_info[0] == 2
|
from collections import OrderedDict # noqa: F401
|
||||||
PY3_ABOVE = sys.version_info[0] >= 3
|
|
||||||
PY26 = PY2 and sys.version_info[1] < 7
|
|
||||||
PY27 = PY2 and sys.version_info[1] == 7
|
|
||||||
PY27_ABOVE = PY27 or PY3_ABOVE
|
|
||||||
|
|
||||||
if PY26:
|
|
||||||
from ordereddict import OrderedDict
|
|
||||||
else:
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from logging import NullHandler
|
from logging import NullHandler
|
||||||
|
@ -37,28 +21,13 @@ except ImportError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
if PY2:
|
text_type = str
|
||||||
from cStringIO import StringIO
|
irange = range
|
||||||
from cStringIO import StringIO as BytesIO
|
PY2 = sys.version[0] == 2
|
||||||
|
|
||||||
text_type = unicode
|
|
||||||
irange = xrange
|
|
||||||
|
|
||||||
class Iterator(object):
|
|
||||||
def next(self):
|
|
||||||
return type(self).__next__(self)
|
|
||||||
|
|
||||||
|
|
||||||
else:
|
|
||||||
from io import StringIO, BytesIO
|
|
||||||
|
|
||||||
text_type = str
|
|
||||||
Iterator = object
|
|
||||||
irange = range
|
|
||||||
|
|
||||||
|
|
||||||
def isstream(instance):
|
def isstream(instance):
|
||||||
""" check if a instance is a stream """
|
"""check if a instance is a stream"""
|
||||||
try:
|
try:
|
||||||
import mmap
|
import mmap
|
||||||
|
|
||||||
|
@ -72,11 +41,4 @@ def isstream(instance):
|
||||||
|
|
||||||
def is_string(atype):
|
def is_string(atype):
|
||||||
"""find out if a type is str or not"""
|
"""find out if a type is str or not"""
|
||||||
if atype == str:
|
return atype == str
|
||||||
return True
|
|
||||||
|
|
||||||
elif PY2:
|
|
||||||
if atype == unicode:
|
|
||||||
return True
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
|
@ -4,13 +4,22 @@
|
||||||
|
|
||||||
The io interface to file extensions
|
The io interface to file extensions
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
|
import warnings
|
||||||
|
|
||||||
import pyexcel_io.manager as manager
|
import pyexcel_io.manager as manager
|
||||||
from pyexcel_io._compact import OrderedDict, isstream, PY2
|
from pyexcel_io._compact import OrderedDict, isstream
|
||||||
|
|
||||||
from .constants import MESSAGE_ERROR_03, MESSAGE_WRONG_IO_INSTANCE
|
from .constants import MESSAGE_ERROR_03, MESSAGE_WRONG_IO_INSTANCE
|
||||||
|
|
||||||
|
DEPRECATED_SINCE_0_6_0 = (
|
||||||
|
"Deprecated since v0.6.0! "
|
||||||
|
+ "Although backward compatibility is preserved, "
|
||||||
|
+ "it is recommended to upgrade to get new features."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RWInterface(object):
|
class RWInterface(object):
|
||||||
"""
|
"""
|
||||||
|
@ -20,6 +29,7 @@ class RWInterface(object):
|
||||||
stream_type = None
|
stream_type = None
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
warnings.warn(DEPRECATED_SINCE_0_6_0)
|
||||||
self._file_type = None
|
self._file_type = None
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
def open(self, file_name, **keywords):
|
||||||
|
@ -85,26 +95,15 @@ class BookReader(RWInterface):
|
||||||
keywords are passed on to individual readers
|
keywords are passed on to individual readers
|
||||||
"""
|
"""
|
||||||
if isstream(file_stream):
|
if isstream(file_stream):
|
||||||
if PY2:
|
from io import UnsupportedOperation
|
||||||
if hasattr(file_stream, "seek"):
|
|
||||||
file_stream.seek(0)
|
|
||||||
else:
|
|
||||||
# python 2
|
|
||||||
# Hei zipfile in odfpy would do a seek
|
|
||||||
# but stream from urlib cannot do seek
|
|
||||||
file_stream = _convert_content_to_stream(
|
|
||||||
file_stream.read(), self._file_type
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
from io import UnsupportedOperation
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
file_stream.seek(0)
|
file_stream.seek(0)
|
||||||
except UnsupportedOperation:
|
except UnsupportedOperation:
|
||||||
# python 3
|
# python 3
|
||||||
file_stream = _convert_content_to_stream(
|
file_stream = _convert_content_to_stream(
|
||||||
file_stream.read(), self._file_type
|
file_stream.read(), self._file_type
|
||||||
)
|
)
|
||||||
|
|
||||||
self._file_stream = file_stream
|
self._file_stream = file_stream
|
||||||
self._keywords = keywords
|
self._keywords = keywords
|
||||||
|
@ -231,6 +230,17 @@ class BookWriter(RWInterface):
|
||||||
|
|
||||||
def _convert_content_to_stream(file_content, file_type):
|
def _convert_content_to_stream(file_content, file_type):
|
||||||
stream = manager.get_io(file_type)
|
stream = manager.get_io(file_type)
|
||||||
|
target_content_type = manager.get_io_type(file_type)
|
||||||
|
needs_encode = target_content_type == "bytes" and not isinstance(
|
||||||
|
file_content, bytes
|
||||||
|
)
|
||||||
|
needs_decode = target_content_type == "string" and isinstance(
|
||||||
|
file_content, bytes
|
||||||
|
)
|
||||||
|
if needs_encode:
|
||||||
|
file_content = file_content.encode("utf-8")
|
||||||
|
elif needs_decode:
|
||||||
|
file_content = file_content.decode("utf-8")
|
||||||
stream.write(file_content)
|
stream.write(file_content)
|
||||||
stream.seek(0)
|
stream.seek(0)
|
||||||
return stream
|
return stream
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
|
|
||||||
Constants appeared in pyexcel
|
Constants appeared in pyexcel
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License
|
:license: New BSD License
|
||||||
"""
|
"""
|
||||||
# flake8: noqa
|
# flake8: noqa
|
||||||
|
@ -16,6 +16,7 @@ MESSAGE_INVALID_PARAMETERS = "Invalid parameters"
|
||||||
MESSAGE_ERROR_02 = "No content, file name. Nothing is given"
|
MESSAGE_ERROR_02 = "No content, file name. Nothing is given"
|
||||||
MESSAGE_ERROR_03 = "cannot handle unknown content"
|
MESSAGE_ERROR_03 = "cannot handle unknown content"
|
||||||
MESSAGE_WRONG_IO_INSTANCE = "Wrong io instance is passed for your file format."
|
MESSAGE_WRONG_IO_INSTANCE = "Wrong io instance is passed for your file format."
|
||||||
|
MESSAGE_FILE_NAME_SHOULD_BE_STRING = "file_name should be a string"
|
||||||
MESSAGE_CANNOT_WRITE_STREAM_FORMATTER = (
|
MESSAGE_CANNOT_WRITE_STREAM_FORMATTER = (
|
||||||
"Cannot write content of file type %s to stream"
|
"Cannot write content of file type %s to stream"
|
||||||
)
|
)
|
||||||
|
@ -31,6 +32,8 @@ MESSAGE_CANNOT_READ_FILE_TYPE_FORMATTER = (
|
||||||
MESSAGE_LOADING_FORMATTER = (
|
MESSAGE_LOADING_FORMATTER = (
|
||||||
"The plugin for file type %s is not installed. Please install %s"
|
"The plugin for file type %s is not installed. Please install %s"
|
||||||
)
|
)
|
||||||
|
MESSAGE_NOT_FILE_FORMATTER = "%s is not a file"
|
||||||
|
MESSAGE_FILE_DOES_NOT_EXIST = "%s does not exist"
|
||||||
MESSAGE_EMPTY_ARRAY = "One empty row is found"
|
MESSAGE_EMPTY_ARRAY = "One empty row is found"
|
||||||
MESSAGE_IGNORE_ROW = "One row is ignored"
|
MESSAGE_IGNORE_ROW = "One row is ignored"
|
||||||
MESSAGE_DB_EXCEPTION = """
|
MESSAGE_DB_EXCEPTION = """
|
||||||
|
@ -44,8 +47,12 @@ FILE_FORMAT_ODS = "ods"
|
||||||
FILE_FORMAT_XLS = "xls"
|
FILE_FORMAT_XLS = "xls"
|
||||||
FILE_FORMAT_XLSX = "xlsx"
|
FILE_FORMAT_XLSX = "xlsx"
|
||||||
FILE_FORMAT_XLSM = "xlsm"
|
FILE_FORMAT_XLSM = "xlsm"
|
||||||
|
FILE_FORMAT_XLSB = "xlsb"
|
||||||
|
FILE_FORMAT_HTML = "html"
|
||||||
|
FILE_FORMAT_PDF = "pdf"
|
||||||
DB_SQL = "sql"
|
DB_SQL = "sql"
|
||||||
DB_DJANGO = "django"
|
DB_DJANGO = "django"
|
||||||
|
DB_QUERYSET = "queryset"
|
||||||
KEYWORD_TSV_DIALECT = "excel-tab"
|
KEYWORD_TSV_DIALECT = "excel-tab"
|
||||||
KEYWORD_LINE_TERMINATOR = "lineterminator"
|
KEYWORD_LINE_TERMINATOR = "lineterminator"
|
||||||
|
|
||||||
|
|
|
@ -4,23 +4,30 @@
|
||||||
|
|
||||||
database data importer and exporter
|
database data importer and exporter
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io.plugins import IOPluginInfoChain
|
from pyexcel_io.plugins import IOPluginInfoChainV2
|
||||||
from pyexcel_io.constants import DB_DJANGO, DB_SQL
|
from pyexcel_io.constants import DB_SQL, DB_DJANGO, DB_QUERYSET
|
||||||
|
|
||||||
|
IOPluginInfoChainV2(__name__).add_a_reader(
|
||||||
IOPluginInfoChain(__name__).add_a_reader(
|
relative_plugin_class_path="exporters.queryset.QueryReader",
|
||||||
|
locations=["file", "memory", "content"],
|
||||||
|
file_types=[DB_QUERYSET],
|
||||||
|
).add_a_reader(
|
||||||
relative_plugin_class_path="exporters.django.DjangoBookReader",
|
relative_plugin_class_path="exporters.django.DjangoBookReader",
|
||||||
|
locations=["file", "memory", "content"],
|
||||||
|
file_types=[DB_DJANGO],
|
||||||
|
).add_a_writer(
|
||||||
|
relative_plugin_class_path="importers.django.DjangoBookWriter",
|
||||||
|
locations=["file", "content", "memory"],
|
||||||
file_types=[DB_DJANGO],
|
file_types=[DB_DJANGO],
|
||||||
).add_a_reader(
|
).add_a_reader(
|
||||||
relative_plugin_class_path="exporters.sqlalchemy.SQLBookReader",
|
relative_plugin_class_path="exporters.sqlalchemy.SQLBookReader",
|
||||||
|
locations=["file", "memory", "content"],
|
||||||
file_types=[DB_SQL],
|
file_types=[DB_SQL],
|
||||||
).add_a_writer(
|
|
||||||
relative_plugin_class_path="importers.django.DjangoBookWriter",
|
|
||||||
file_types=[DB_DJANGO],
|
|
||||||
).add_a_writer(
|
).add_a_writer(
|
||||||
relative_plugin_class_path="importers.sqlalchemy.SQLBookWriter",
|
relative_plugin_class_path="importers.sqlalchemy.SQLBookWriter",
|
||||||
|
locations=["file", "content", "memory"],
|
||||||
file_types=[DB_SQL],
|
file_types=[DB_SQL],
|
||||||
)
|
)
|
||||||
|
|
|
@ -4,31 +4,13 @@
|
||||||
|
|
||||||
Common classes shared among database importers and exporters
|
Common classes shared among database importers and exporters
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io.book import BookReader
|
|
||||||
|
|
||||||
|
|
||||||
class DbExporter(BookReader):
|
|
||||||
""" Transcode the book reader interface to db interface """
|
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
|
||||||
self.export_tables(self, file_name, **keywords)
|
|
||||||
|
|
||||||
def open_stream(self, file_stream, **keywords):
|
|
||||||
self.export_tables(self, file_stream, **keywords)
|
|
||||||
|
|
||||||
def open_content(self, file_content, **keywords):
|
|
||||||
self.export_tables(file_content, **keywords)
|
|
||||||
|
|
||||||
def export_tables(self, exporter, **keywords):
|
|
||||||
""" read database tables """
|
|
||||||
raise NotImplementedError("Please implement this method")
|
|
||||||
|
|
||||||
|
|
||||||
class DjangoModelExportAdapter(object):
|
class DjangoModelExportAdapter(object):
|
||||||
""" django export parameter holder """
|
"""django export parameter holder"""
|
||||||
|
|
||||||
def __init__(self, model, export_columns=None):
|
def __init__(self, model, export_columns=None):
|
||||||
self.model = model
|
self.model = model
|
||||||
|
@ -36,19 +18,19 @@ class DjangoModelExportAdapter(object):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def name(self):
|
def name(self):
|
||||||
""" get database table name """
|
"""get database table name"""
|
||||||
return self.get_name()
|
return self.get_name()
|
||||||
|
|
||||||
def get_name(self):
|
def get_name(self):
|
||||||
""" get database table name """
|
"""get database table name"""
|
||||||
return self.model._meta.model_name
|
return self.model._meta.model_name
|
||||||
|
|
||||||
|
|
||||||
class DjangoModelImportAdapter(DjangoModelExportAdapter):
|
class DjangoModelImportAdapter(DjangoModelExportAdapter):
|
||||||
""" parameter holder for django data import """
|
"""parameter holder for django data import"""
|
||||||
|
|
||||||
class InOutParameter(object):
|
class InOutParameter(object):
|
||||||
""" local class to manipulate variable io """
|
"""local class to manipulate variable io"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.output = None
|
self.output = None
|
||||||
|
@ -56,91 +38,96 @@ class DjangoModelImportAdapter(DjangoModelExportAdapter):
|
||||||
|
|
||||||
def __init__(self, model):
|
def __init__(self, model):
|
||||||
DjangoModelExportAdapter.__init__(self, model)
|
DjangoModelExportAdapter.__init__(self, model)
|
||||||
self.__column_names = self.InOutParameter()
|
self._column_names = self.InOutParameter()
|
||||||
self.__column_name_mapping_dict = self.InOutParameter()
|
self._column_name_mapping_dict = self.InOutParameter()
|
||||||
self.__row_initializer = self.InOutParameter()
|
self._row_initializer = self.InOutParameter()
|
||||||
self._process_parameters()
|
self._process_parameters()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def row_initializer(self):
|
def row_initializer(self):
|
||||||
""" contructor for a database table entry """
|
"""contructor for a database table entry"""
|
||||||
return self.__row_initializer.output
|
return self._row_initializer.output
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def column_names(self):
|
def column_names(self):
|
||||||
""" the desginated database column names """
|
"""the desginated database column names"""
|
||||||
return self.__column_names.output
|
return self._column_names.output
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def column_name_mapping_dict(self):
|
def column_name_mapping_dict(self):
|
||||||
""" if not the same, a mapping dictionary is looked up"""
|
"""if not the same, a mapping dictionary is looked up"""
|
||||||
return self.__column_name_mapping_dict.output
|
return self._column_name_mapping_dict.output
|
||||||
|
|
||||||
@row_initializer.setter
|
@row_initializer.setter
|
||||||
def row_initializer(self, a_function):
|
def row_initializer(self, a_function):
|
||||||
""" set the contructor """
|
"""set the contructor"""
|
||||||
self.__row_initializer.input = a_function
|
self._row_initializer.input = a_function
|
||||||
self._process_parameters()
|
self._process_parameters()
|
||||||
|
|
||||||
@column_names.setter
|
@column_names.setter
|
||||||
def column_names(self, column_names):
|
def column_names(self, column_names):
|
||||||
""" set the column names """
|
"""set the column names"""
|
||||||
self.__column_names.input = column_names
|
self._column_names.input = column_names
|
||||||
self._process_parameters()
|
self._process_parameters()
|
||||||
|
|
||||||
@column_name_mapping_dict.setter
|
@column_name_mapping_dict.setter
|
||||||
def column_name_mapping_dict(self, mapping_dict):
|
def column_name_mapping_dict(self, mapping_dict):
|
||||||
""" set the mapping dict """
|
"""set the mapping dict"""
|
||||||
self.__column_name_mapping_dict.input = mapping_dict
|
self._column_name_mapping_dict.input = mapping_dict
|
||||||
self._process_parameters()
|
self._process_parameters()
|
||||||
|
|
||||||
def _process_parameters(self):
|
def _process_parameters(self):
|
||||||
if self.__row_initializer.input is None:
|
if self._row_initializer.input is None:
|
||||||
self.__row_initializer.output = None
|
self._row_initializer.output = None
|
||||||
else:
|
else:
|
||||||
self.__row_initializer.output = self.__row_initializer.input
|
self._row_initializer.output = self._row_initializer.input
|
||||||
if isinstance(self.__column_name_mapping_dict.input, list):
|
if isinstance(self._column_name_mapping_dict.input, list):
|
||||||
self.__column_names.output = self.__column_name_mapping_dict.input
|
self._column_names.output = self._column_name_mapping_dict.input
|
||||||
self.__column_name_mapping_dict.output = None
|
self._column_name_mapping_dict.output = None
|
||||||
elif isinstance(self.__column_name_mapping_dict.input, dict):
|
elif isinstance(self._column_name_mapping_dict.input, dict):
|
||||||
if self.__column_names.input:
|
|
||||||
self.__column_names.output = [
|
if self._column_names.input:
|
||||||
self.__column_name_mapping_dict.input[name]
|
self._column_names.output = []
|
||||||
for name in self.__column_names.input
|
indices = []
|
||||||
]
|
for index, name in enumerate(self._column_names.input):
|
||||||
self.__column_name_mapping_dict.output = None
|
if name in self._column_name_mapping_dict.input:
|
||||||
if self.__column_names.output is None:
|
self._column_names.output.append(
|
||||||
self.__column_names.output = self.__column_names.input
|
self._column_name_mapping_dict.input[name]
|
||||||
|
)
|
||||||
|
indices.append(index)
|
||||||
|
self._column_name_mapping_dict.output = indices
|
||||||
|
if self._column_names.output is None:
|
||||||
|
self._column_names.output = self._column_names.input
|
||||||
|
|
||||||
|
|
||||||
class DjangoModelExporter(object):
|
class DjangoModelExporter(object):
|
||||||
""" public interface for django model export """
|
"""public interface for django model export"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.adapters = []
|
self.adapters = []
|
||||||
|
|
||||||
def append(self, import_adapter):
|
def append(self, import_adapter):
|
||||||
""" store model parameter for more than one model """
|
"""store model parameter for more than one model"""
|
||||||
self.adapters.append(import_adapter)
|
self.adapters.append(import_adapter)
|
||||||
|
|
||||||
|
|
||||||
class DjangoModelImporter(object):
|
class DjangoModelImporter(object):
|
||||||
""" public interface for django model import """
|
"""public interface for django model import"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.__adapters = {}
|
self._adapters = {}
|
||||||
|
|
||||||
def append(self, import_adapter):
|
def append(self, import_adapter):
|
||||||
""" store model parameter for more than one model """
|
"""store model parameter for more than one model"""
|
||||||
self.__adapters[import_adapter.get_name()] = import_adapter
|
self._adapters[import_adapter.get_name()] = import_adapter
|
||||||
|
|
||||||
def get(self, name):
|
def get(self, name):
|
||||||
""" get a parameter out """
|
"""get a parameter out"""
|
||||||
return self.__adapters.get(name, None)
|
return self._adapters.get(name, None)
|
||||||
|
|
||||||
|
|
||||||
class SQLTableExportAdapter(DjangoModelExportAdapter):
|
class SQLTableExportAdapter(DjangoModelExportAdapter):
|
||||||
""" parameter holder for sql table data export """
|
"""parameter holder for sql table data export"""
|
||||||
|
|
||||||
def __init__(self, model, export_columns=None):
|
def __init__(self, model, export_columns=None):
|
||||||
DjangoModelExportAdapter.__init__(self, model, export_columns)
|
DjangoModelExportAdapter.__init__(self, model, export_columns)
|
||||||
|
@ -151,7 +138,7 @@ class SQLTableExportAdapter(DjangoModelExportAdapter):
|
||||||
|
|
||||||
|
|
||||||
class SQLTableImportAdapter(DjangoModelImportAdapter):
|
class SQLTableImportAdapter(DjangoModelImportAdapter):
|
||||||
""" parameter holder for sqlalchemy table import """
|
"""parameter holder for sqlalchemy table import"""
|
||||||
|
|
||||||
def __init__(self, model):
|
def __init__(self, model):
|
||||||
DjangoModelImportAdapter.__init__(self, model)
|
DjangoModelImportAdapter.__init__(self, model)
|
||||||
|
@ -162,7 +149,7 @@ class SQLTableImportAdapter(DjangoModelImportAdapter):
|
||||||
|
|
||||||
|
|
||||||
class SQLTableExporter(DjangoModelExporter):
|
class SQLTableExporter(DjangoModelExporter):
|
||||||
""" public interface for sql table export """
|
"""public interface for sql table export"""
|
||||||
|
|
||||||
def __init__(self, session):
|
def __init__(self, session):
|
||||||
DjangoModelExporter.__init__(self)
|
DjangoModelExporter.__init__(self)
|
||||||
|
@ -170,7 +157,7 @@ class SQLTableExporter(DjangoModelExporter):
|
||||||
|
|
||||||
|
|
||||||
class SQLTableImporter(DjangoModelImporter):
|
class SQLTableImporter(DjangoModelImporter):
|
||||||
""" public interface to do data import via sqlalchemy """
|
"""public interface to do data import via sqlalchemy"""
|
||||||
|
|
||||||
def __init__(self, session):
|
def __init__(self, session):
|
||||||
DjangoModelImporter.__init__(self)
|
DjangoModelImporter.__init__(self)
|
||||||
|
|
|
@ -4,16 +4,15 @@
|
||||||
|
|
||||||
The lower level handler for django import and export
|
The lower level handler for django import and export
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io.database.common import DbExporter
|
from pyexcel_io.plugin_api import IReader
|
||||||
from pyexcel_io.database.querysets import QuerysetsReader
|
from pyexcel_io.database.querysets import QuerysetsReader
|
||||||
|
|
||||||
|
|
||||||
class DjangoModelReader(QuerysetsReader):
|
class DjangoModelReader(QuerysetsReader):
|
||||||
"""Read from django model
|
"""Read from django model"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, model, export_columns=None, **keywords):
|
def __init__(self, model, export_columns=None, **keywords):
|
||||||
self.__model = model
|
self.__model = model
|
||||||
|
@ -28,22 +27,20 @@ class DjangoModelReader(QuerysetsReader):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class DjangoBookReader(DbExporter):
|
class DjangoBookReader(IReader):
|
||||||
""" read django models """
|
"""read django models"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, exporter, _, **keywords):
|
||||||
DbExporter.__init__(self)
|
self.exporter = exporter
|
||||||
self.exporter = None
|
self.keywords = keywords
|
||||||
|
self.content_array = self.exporter.adapters
|
||||||
|
|
||||||
def export_tables(self, file_content, **keywords):
|
def read_sheet(self, native_sheet_index):
|
||||||
self.exporter = file_content
|
native_sheet = self.content_array[native_sheet_index]
|
||||||
self._load_from_django_models()
|
|
||||||
|
|
||||||
def read_sheet(self, native_sheet):
|
|
||||||
reader = DjangoModelReader(
|
reader = DjangoModelReader(
|
||||||
native_sheet.model, native_sheet.export_columns
|
native_sheet.model, export_columns=native_sheet.export_columns
|
||||||
)
|
)
|
||||||
return reader.to_array()
|
return reader
|
||||||
|
|
||||||
def _load_from_django_models(self):
|
def close(self):
|
||||||
self._native_book = self.exporter.adapters
|
pass
|
||||||
|
|
|
@ -0,0 +1,20 @@
|
||||||
|
from pyexcel_io.plugin_api import IReader
|
||||||
|
from pyexcel_io.database.querysets import QuerysetsReader
|
||||||
|
|
||||||
|
|
||||||
|
class QueryReader(IReader):
|
||||||
|
def __init__(self, query_sets, _, column_names=None, **keywords):
|
||||||
|
self.query_sets = query_sets
|
||||||
|
self.column_names = column_names
|
||||||
|
self.keywords = keywords
|
||||||
|
self.content_array = [
|
||||||
|
QuerysetsReader(
|
||||||
|
self.query_sets, self.column_names, **self.keywords
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
def read_sheet(self, index):
|
||||||
|
return self.content_array[index]
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
|
@ -4,16 +4,15 @@
|
||||||
|
|
||||||
The lower level handler for database import and export
|
The lower level handler for database import and export
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io.database.common import DbExporter
|
from pyexcel_io.plugin_api import IReader
|
||||||
from pyexcel_io.database.querysets import QuerysetsReader
|
from pyexcel_io.database.querysets import QuerysetsReader
|
||||||
|
|
||||||
|
|
||||||
class SQLTableReader(QuerysetsReader):
|
class SQLTableReader(QuerysetsReader):
|
||||||
"""Read a table
|
"""Read a table"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, session, table, export_columns=None, **keywords):
|
def __init__(self, session, table, export_columns=None, **keywords):
|
||||||
everything = session.query(table).all()
|
everything = session.query(table).all()
|
||||||
|
@ -32,24 +31,22 @@ class SQLTableReader(QuerysetsReader):
|
||||||
QuerysetsReader.__init__(self, everything, column_names, **keywords)
|
QuerysetsReader.__init__(self, everything, column_names, **keywords)
|
||||||
|
|
||||||
|
|
||||||
class SQLBookReader(DbExporter):
|
class SQLBookReader(IReader):
|
||||||
""" read a table via sqlalchemy """
|
"""read a table via sqlalchemy"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, exporter, _, **keywords):
|
||||||
DbExporter.__init__(self)
|
self.__exporter = exporter
|
||||||
self.__exporter = None
|
self.content_array = self.__exporter.adapters
|
||||||
|
self.keywords = keywords
|
||||||
|
|
||||||
def export_tables(self, file_content, **keywords):
|
def read_sheet(self, native_sheet_index):
|
||||||
self.__exporter = file_content
|
native_sheet = self.content_array[native_sheet_index]
|
||||||
self._load_from_tables()
|
|
||||||
|
|
||||||
def read_sheet(self, native_sheet):
|
|
||||||
reader = SQLTableReader(
|
reader = SQLTableReader(
|
||||||
self.__exporter.session,
|
self.__exporter.session,
|
||||||
native_sheet.table,
|
native_sheet.table,
|
||||||
native_sheet.export_columns,
|
native_sheet.export_columns,
|
||||||
)
|
)
|
||||||
return reader.to_array()
|
return reader
|
||||||
|
|
||||||
def _load_from_tables(self):
|
def close(self):
|
||||||
self._native_book = self.__exporter.adapters
|
pass
|
||||||
|
|
|
@ -4,77 +4,75 @@
|
||||||
|
|
||||||
The lower level handler for django import and export
|
The lower level handler for django import and export
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from pyexcel_io.book import BookWriter
|
|
||||||
from pyexcel_io.sheet import SheetWriter
|
|
||||||
from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none
|
|
||||||
import pyexcel_io.constants as constants
|
import pyexcel_io.constants as constants
|
||||||
|
from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none
|
||||||
|
from pyexcel_io.plugin_api import IWriter, ISheetWriter
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class DjangoModelWriter(SheetWriter):
|
class DjangoModelWriter(ISheetWriter):
|
||||||
""" import data into a django model """
|
"""import data into a django model"""
|
||||||
|
|
||||||
def __init__(self, importer, adapter, batch_size=None, bulk_save=True):
|
def __init__(self, importer, adapter, batch_size=None, bulk_save=True):
|
||||||
SheetWriter.__init__(self, importer, adapter, adapter.name)
|
self.batch_size = batch_size
|
||||||
self.__batch_size = batch_size
|
self.model = adapter.model
|
||||||
self.__model = adapter.model
|
self.column_names = adapter.column_names
|
||||||
self.__column_names = adapter.column_names
|
self.mapdict = adapter.column_name_mapping_dict
|
||||||
self.__mapdict = adapter.column_name_mapping_dict
|
self.initializer = adapter.row_initializer
|
||||||
self.__initializer = adapter.row_initializer
|
self.objs = []
|
||||||
self.__objs = []
|
self.bulk_save = bulk_save
|
||||||
self.__bulk_save = bulk_save
|
self.adapter = adapter
|
||||||
|
|
||||||
def write_row(self, array):
|
def write_row(self, array):
|
||||||
if is_empty_array(array):
|
if is_empty_array(array):
|
||||||
print(constants.MESSAGE_EMPTY_ARRAY)
|
log.warning(constants.MESSAGE_EMPTY_ARRAY)
|
||||||
else:
|
else:
|
||||||
new_array = swap_empty_string_for_none(array)
|
new_array = swap_empty_string_for_none(array)
|
||||||
|
if self.mapdict:
|
||||||
|
another_new_array = []
|
||||||
|
for index, element in enumerate(new_array):
|
||||||
|
if index in self.mapdict:
|
||||||
|
another_new_array.append(element)
|
||||||
|
new_array = another_new_array
|
||||||
model_to_be_created = new_array
|
model_to_be_created = new_array
|
||||||
if self.__initializer is not None:
|
if self.initializer is not None:
|
||||||
model_to_be_created = self.__initializer(new_array)
|
model_to_be_created = self.initializer(new_array)
|
||||||
if model_to_be_created:
|
if model_to_be_created:
|
||||||
self.__objs.append(
|
row = dict(zip(self.column_names, model_to_be_created))
|
||||||
self.__model(
|
self.objs.append(self.model(**row))
|
||||||
**dict(zip(self.__column_names, model_to_be_created))
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
# else
|
# else
|
||||||
# skip the row
|
# skip the row
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
if self.__bulk_save:
|
if self.bulk_save:
|
||||||
self.__model.objects.bulk_create(
|
self.model.objects.bulk_create(
|
||||||
self.__objs, batch_size=self.__batch_size
|
self.objs, batch_size=self.batch_size
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
for an_object in self.__objs:
|
for an_object in self.objs:
|
||||||
an_object.save()
|
an_object.save()
|
||||||
|
|
||||||
|
|
||||||
class DjangoBookWriter(BookWriter):
|
class DjangoBookWriter(IWriter):
|
||||||
""" write data into django models """
|
"""write data into django models"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, exporter, _, **keywords):
|
||||||
BookWriter.__init__(self)
|
self.importer = exporter
|
||||||
self.__importer = None
|
|
||||||
|
|
||||||
def open_content(self, file_content, **keywords):
|
|
||||||
self.__importer = file_content
|
|
||||||
self._keywords = keywords
|
self._keywords = keywords
|
||||||
|
|
||||||
def create_sheet(self, sheet_name):
|
def create_sheet(self, sheet_name):
|
||||||
sheet_writer = None
|
sheet_writer = None
|
||||||
model = self.__importer.get(sheet_name)
|
model = self.importer.get(sheet_name)
|
||||||
if model:
|
if model:
|
||||||
sheet_writer = DjangoModelWriter(
|
sheet_writer = DjangoModelWriter(
|
||||||
self.__importer,
|
self.importer,
|
||||||
model,
|
model,
|
||||||
batch_size=self._keywords.get("batch_size", None),
|
batch_size=self._keywords.get("batch_size", None),
|
||||||
bulk_save=self._keywords.get("bulk_save", True),
|
bulk_save=self._keywords.get("bulk_save", True),
|
||||||
|
@ -86,3 +84,6 @@ class DjangoBookWriter(BookWriter):
|
||||||
)
|
)
|
||||||
|
|
||||||
return sheet_writer
|
return sheet_writer
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
|
@ -4,13 +4,16 @@
|
||||||
|
|
||||||
The lower level handler for database import and export
|
The lower level handler for database import and export
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io.book import BookWriter
|
import logging
|
||||||
from pyexcel_io.sheet import SheetWriter
|
|
||||||
from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none
|
|
||||||
import pyexcel_io.constants as constants
|
import pyexcel_io.constants as constants
|
||||||
|
from pyexcel_io.utils import is_empty_array, swap_empty_string_for_none
|
||||||
|
from pyexcel_io.plugin_api import IWriter, ISheetWriter
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class PyexcelSQLSkipRowException(Exception):
|
class PyexcelSQLSkipRowException(Exception):
|
||||||
|
@ -22,66 +25,62 @@ class PyexcelSQLSkipRowException(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class SQLTableWriter(SheetWriter):
|
class SQLTableWriter(ISheetWriter):
|
||||||
"""Write to a table
|
"""Write to a table"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self, importer, adapter, auto_commit=True, bulk_size=1000, **keywords
|
self, importer, adapter, auto_commit=True, bulk_size=1000, **keywords
|
||||||
):
|
):
|
||||||
SheetWriter.__init__(
|
|
||||||
self, importer, adapter, adapter.get_name(), **keywords
|
|
||||||
)
|
|
||||||
self.__auto_commit = auto_commit
|
self.__auto_commit = auto_commit
|
||||||
self.__count = 0
|
self.__count = 0
|
||||||
self.__bulk_size = bulk_size
|
self.__bulk_size = bulk_size
|
||||||
|
self.adapter = adapter
|
||||||
|
self.importer = importer
|
||||||
|
|
||||||
def write_row(self, array):
|
def write_row(self, array):
|
||||||
if is_empty_array(array):
|
if is_empty_array(array):
|
||||||
print(constants.MESSAGE_EMPTY_ARRAY)
|
LOG.warning(constants.MESSAGE_EMPTY_ARRAY)
|
||||||
else:
|
else:
|
||||||
new_array = swap_empty_string_for_none(array)
|
new_array = swap_empty_string_for_none(array)
|
||||||
try:
|
try:
|
||||||
self._write_row(new_array)
|
self._write_row(new_array)
|
||||||
except PyexcelSQLSkipRowException:
|
except PyexcelSQLSkipRowException:
|
||||||
print(constants.MESSAGE_IGNORE_ROW)
|
LOG.info(constants.MESSAGE_IGNORE_ROW)
|
||||||
print(new_array)
|
LOG.info(new_array)
|
||||||
|
|
||||||
def _write_row(self, array):
|
def _write_row(self, array):
|
||||||
row = dict(zip(self._native_sheet.column_names, array))
|
new_array = array
|
||||||
|
if self.adapter.column_name_mapping_dict:
|
||||||
|
another_new_array = []
|
||||||
|
for index, element in enumerate(new_array):
|
||||||
|
if index in self.adapter.column_name_mapping_dict:
|
||||||
|
another_new_array.append(element)
|
||||||
|
new_array = another_new_array
|
||||||
|
row = dict(zip(self.adapter.column_names, new_array))
|
||||||
obj = None
|
obj = None
|
||||||
if self._native_sheet.row_initializer:
|
if self.adapter.row_initializer:
|
||||||
# allow initinalizer to return None
|
# allow initinalizer to return None
|
||||||
# if skipping is needed
|
# if skipping is needed
|
||||||
obj = self._native_sheet.row_initializer(row)
|
obj = self.adapter.row_initializer(row)
|
||||||
if obj is None:
|
if obj is None:
|
||||||
obj = self._native_sheet.table()
|
obj = self.adapter.table()
|
||||||
for name in self._native_sheet.column_names:
|
for name in self.adapter.column_names:
|
||||||
if self._native_sheet.column_name_mapping_dict is not None:
|
setattr(obj, name, row[name])
|
||||||
key = self._native_sheet.column_name_mapping_dict[name]
|
self.importer.session.add(obj)
|
||||||
else:
|
|
||||||
key = name
|
|
||||||
setattr(obj, key, row[name])
|
|
||||||
self._native_book.session.add(obj)
|
|
||||||
if self.__auto_commit and self.__bulk_size != float("inf"):
|
if self.__auto_commit and self.__bulk_size != float("inf"):
|
||||||
self.__count += 1
|
self.__count += 1
|
||||||
if self.__count % self.__bulk_size == 0:
|
if self.__count % self.__bulk_size == 0:
|
||||||
self._native_book.session.commit()
|
self.importer.session.commit()
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
if self.__auto_commit:
|
if self.__auto_commit:
|
||||||
self._native_book.session.commit()
|
self.importer.session.commit()
|
||||||
|
|
||||||
|
|
||||||
class SQLBookWriter(BookWriter):
|
class SQLBookWriter(IWriter):
|
||||||
""" write data into database tables via sqlalchemy """
|
"""write data into database tables via sqlalchemy"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, file_content, _, auto_commit=True, **keywords):
|
||||||
BookWriter.__init__(self)
|
|
||||||
self.__importer = None
|
|
||||||
self.__auto_commit = True
|
|
||||||
|
|
||||||
def open_content(self, file_content, auto_commit=True, **keywords):
|
|
||||||
self.__importer = file_content
|
self.__importer = file_content
|
||||||
self.__auto_commit = auto_commit
|
self.__auto_commit = auto_commit
|
||||||
|
|
||||||
|
@ -99,3 +98,6 @@ class SQLBookWriter(BookWriter):
|
||||||
)
|
)
|
||||||
|
|
||||||
return sheet_writer
|
return sheet_writer
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
|
@ -4,23 +4,27 @@
|
||||||
|
|
||||||
The lower level handler for querysets
|
The lower level handler for querysets
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
import datetime
|
import datetime
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
|
|
||||||
from pyexcel_io.sheet import SheetReader
|
from pyexcel_io.constants import DEFAULT_SHEET_NAME
|
||||||
|
from pyexcel_io.plugin_api.abstract_sheet import ISheet
|
||||||
|
|
||||||
|
|
||||||
class QuerysetsReader(SheetReader):
|
class QuerysetsReader(ISheet):
|
||||||
""" turn querysets into an array """
|
"""turn querysets into an array"""
|
||||||
|
|
||||||
def __init__(self, query_sets, column_names, **keywords):
|
def __init__(self, query_sets, column_names):
|
||||||
SheetReader.__init__(self, query_sets, **keywords)
|
self.name = DEFAULT_SHEET_NAME
|
||||||
self.__column_names = column_names
|
self.__column_names = column_names
|
||||||
self.__query_sets = query_sets
|
self.__query_sets = query_sets
|
||||||
|
|
||||||
|
def row_iterator(self):
|
||||||
|
return chain([self.__column_names], self.__query_sets)
|
||||||
|
|
||||||
def to_array(self):
|
def to_array(self):
|
||||||
"""
|
"""
|
||||||
Convert query sets into an array
|
Convert query sets into an array
|
||||||
|
@ -28,11 +32,11 @@ class QuerysetsReader(SheetReader):
|
||||||
if len(self.__query_sets) == 0:
|
if len(self.__query_sets) == 0:
|
||||||
yield []
|
yield []
|
||||||
|
|
||||||
for element in SheetReader.to_array(self):
|
for row in self.row_iterator():
|
||||||
yield element
|
row_values = []
|
||||||
|
for value in self.column_iterator(row):
|
||||||
def row_iterator(self):
|
row_values.append(value)
|
||||||
return chain([self.__column_names], self.__query_sets)
|
yield row_values
|
||||||
|
|
||||||
def column_iterator(self, row):
|
def column_iterator(self, row):
|
||||||
if self.__column_names is None:
|
if self.__column_names is None:
|
||||||
|
@ -52,7 +56,7 @@ class QuerysetsReader(SheetReader):
|
||||||
|
|
||||||
|
|
||||||
def get_complex_attribute(row, attribute):
|
def get_complex_attribute(row, attribute):
|
||||||
""" recursively get an attribute """
|
"""recursively get an attribute"""
|
||||||
attributes = attribute.split("__")
|
attributes = attribute.split("__")
|
||||||
value = row
|
value = row
|
||||||
try:
|
try:
|
||||||
|
@ -64,7 +68,7 @@ def get_complex_attribute(row, attribute):
|
||||||
|
|
||||||
|
|
||||||
def get_simple_attribute(row, attribute):
|
def get_simple_attribute(row, attribute):
|
||||||
""" get dotted attribute """
|
"""get dotted attribute"""
|
||||||
value = getattr(row, attribute)
|
value = getattr(row, attribute)
|
||||||
if isinstance(value, (datetime.date, datetime.time)):
|
if isinstance(value, (datetime.date, datetime.time)):
|
||||||
value = value.isoformat()
|
value = value.isoformat()
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
|
|
||||||
all possible exceptions
|
all possible exceptions
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@ -21,12 +21,6 @@ class SupportingPluginAvailableButNotInstalled(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class UpgradePlugin(Exception):
|
|
||||||
"""raised when a known plugin is not compatible"""
|
|
||||||
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class IntegerAccuracyLossError(Exception):
|
class IntegerAccuracyLossError(Exception):
|
||||||
"""
|
"""
|
||||||
When an interger is greater than 999999999999999, ods loses its accuracy.
|
When an interger is greater than 999999999999999, ods loses its accuracy.
|
||||||
|
@ -46,11 +40,13 @@ class IntegerAccuracyLossError(Exception):
|
||||||
b=get_sheet(file_name='abc.ods')
|
b=get_sheet(file_name='abc.ods')
|
||||||
b[0,0] != s[0,0]
|
b[0,0] != s[0,0]
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, message):
|
def __init__(self, message):
|
||||||
custom_message = (
|
custom_message = (
|
||||||
message + '\n' +
|
message
|
||||||
"In order to keep its accuracy, please save as string. Then " +
|
+ "\n"
|
||||||
"convert to int, long or float after the value will be read back"
|
+ "In order to keep its accuracy, please save as string. Then "
|
||||||
|
+ "convert to int, long or float after the value will be read back"
|
||||||
)
|
)
|
||||||
|
|
||||||
super(IntegerAccuracyLossError, self).__init__(custom_message)
|
super(IntegerAccuracyLossError, self).__init__(custom_message)
|
||||||
|
|
198
pyexcel_io/io.py
198
pyexcel_io/io.py
|
@ -4,20 +4,60 @@
|
||||||
|
|
||||||
The io interface to file extensions
|
The io interface to file extensions
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from types import GeneratorType
|
import os
|
||||||
import warnings
|
import warnings
|
||||||
|
from types import GeneratorType
|
||||||
|
|
||||||
from pyexcel_io._compact import isstream, PY2
|
from pyexcel_io import constants
|
||||||
from pyexcel_io.plugins import READERS, WRITERS
|
from pyexcel_io.reader import Reader
|
||||||
import pyexcel_io.constants as constants
|
from pyexcel_io.writer import Writer
|
||||||
|
from pyexcel_io.plugins import OLD_READERS, OLD_WRITERS
|
||||||
|
from pyexcel_io._compact import isstream
|
||||||
|
from pyexcel_io.exceptions import (
|
||||||
|
NoSupportingPluginFound,
|
||||||
|
SupportingPluginAvailableButNotInstalled,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def iget_data(afile, file_type=None, **keywords):
|
def iget_data(afile, file_type=None, **keywords):
|
||||||
"""Get data from an excel file source
|
"""Get data from an excel file source
|
||||||
|
|
||||||
|
The data has not gone into memory yet. If you use dedicated partial read
|
||||||
|
plugins, such as pyexcel-xlsxr, pyexcel-odsr, you will notice
|
||||||
|
the memory consumption drop when you work with big files.
|
||||||
|
|
||||||
|
:param afile: a file name, a file stream or actual content
|
||||||
|
:param sheet_name: the name of the sheet to be loaded
|
||||||
|
:param sheet_index: the index of the sheet to be loaded
|
||||||
|
:param sheets: a list of sheet to be loaded
|
||||||
|
:param file_type: used only when filename is not a physical file name
|
||||||
|
:param force_file_type: used only when filename refers to a physical file
|
||||||
|
and it is intended to open it as forced file type.
|
||||||
|
:param library: explicitly name a library for use.
|
||||||
|
e.g. library='pyexcel-ods'
|
||||||
|
:param auto_detect_float: defaults to True
|
||||||
|
:param auto_detect_int: defaults to True
|
||||||
|
:param auto_detect_datetime: defaults to True
|
||||||
|
:param ignore_infinity: defaults to True
|
||||||
|
:param ignore_nan_text: various forms of 'NaN', 'nan' are ignored
|
||||||
|
:param default_float_nan: choose one form of 'NaN', 'nan'
|
||||||
|
:param pep_0515_off: turn off pep 0515. default to True.
|
||||||
|
:param keep_trailing_empty_cells: keep trailing columns. default to False
|
||||||
|
:param keywords: any other library specific parameters
|
||||||
|
:returns: an ordered dictionary
|
||||||
|
"""
|
||||||
|
data, reader = _get_data(
|
||||||
|
afile, file_type=file_type, streaming=True, **keywords
|
||||||
|
)
|
||||||
|
return data, reader
|
||||||
|
|
||||||
|
|
||||||
|
def get_data(afile, file_type=None, streaming=None, **keywords):
|
||||||
|
"""Get data from an excel file source
|
||||||
|
|
||||||
:param afile: a file name, a file stream or actual content
|
:param afile: a file name, a file stream or actual content
|
||||||
:param sheet_name: the name of the sheet to be loaded
|
:param sheet_name: the name of the sheet to be loaded
|
||||||
:param sheet_index: the index of the sheet to be loaded
|
:param sheet_index: the index of the sheet to be loaded
|
||||||
|
@ -37,31 +77,7 @@ def iget_data(afile, file_type=None, **keywords):
|
||||||
:param ignore_nan_text: various forms of 'NaN', 'nan' are ignored
|
:param ignore_nan_text: various forms of 'NaN', 'nan' are ignored
|
||||||
:param default_float_nan: choose one form of 'NaN', 'nan'
|
:param default_float_nan: choose one form of 'NaN', 'nan'
|
||||||
:param pep_0515_off: turn off pep 0515. default to True.
|
:param pep_0515_off: turn off pep 0515. default to True.
|
||||||
:param keywords: any other library specific parameters
|
:param keep_trailing_empty_cells: keep trailing columns. default to False
|
||||||
:returns: an ordered dictionary
|
|
||||||
"""
|
|
||||||
data, reader = _get_data(
|
|
||||||
afile, file_type=file_type, streaming=True, **keywords
|
|
||||||
)
|
|
||||||
return data, reader
|
|
||||||
|
|
||||||
|
|
||||||
def get_data(afile, file_type=None, streaming=None, **keywords):
|
|
||||||
"""Get data from an excel file source
|
|
||||||
|
|
||||||
:param afile: a file name, a file stream or actual content
|
|
||||||
:param sheet_name: the name of the sheet to be loaded
|
|
||||||
:param sheet_index: the index of the sheet to be loaded
|
|
||||||
:param file_type: used only when filename is not a physial file name
|
|
||||||
:param streaming: toggles the type of returned data. The values of the
|
|
||||||
returned dictionary remain as generator if it is set
|
|
||||||
to True. Default is False.
|
|
||||||
:param library: explicitly name a library for use.
|
|
||||||
e.g. library='pyexcel-ods'
|
|
||||||
:param auto_detect_float: defaults to True
|
|
||||||
:param auto_detect_int: defaults to True
|
|
||||||
:param auto_detect_datetime: defaults to True
|
|
||||||
:param ignore_infinity: defaults to True
|
|
||||||
:param keywords: any other library specific parameters
|
:param keywords: any other library specific parameters
|
||||||
:returns: an ordered dictionary
|
:returns: an ordered dictionary
|
||||||
"""
|
"""
|
||||||
|
@ -97,6 +113,8 @@ def save_data(afile, data, file_type=None, **keywords):
|
||||||
:param filename: actual file name, a file stream or actual content
|
:param filename: actual file name, a file stream or actual content
|
||||||
:param data: a dictionary but an ordered dictionary is preferred
|
:param data: a dictionary but an ordered dictionary is preferred
|
||||||
:param file_type: used only when filename is not a physial file name
|
:param file_type: used only when filename is not a physial file name
|
||||||
|
:param force_file_type: used only when filename refers to a physical file
|
||||||
|
and it is intended to open it as forced file type.
|
||||||
:param library: explicitly name a library for use.
|
:param library: explicitly name a library for use.
|
||||||
e.g. library='pyexcel-ods'
|
e.g. library='pyexcel-ods'
|
||||||
:param keywords: any other parameters that python csv module's
|
:param keywords: any other parameters that python csv module's
|
||||||
|
@ -109,39 +127,20 @@ def save_data(afile, data, file_type=None, **keywords):
|
||||||
single_sheet_in_book = True
|
single_sheet_in_book = True
|
||||||
to_store = {constants.DEFAULT_SHEET_NAME: data}
|
to_store = {constants.DEFAULT_SHEET_NAME: data}
|
||||||
else:
|
else:
|
||||||
if PY2:
|
keys = list(data.keys())
|
||||||
keys = data.keys()
|
|
||||||
else:
|
|
||||||
keys = list(data.keys())
|
|
||||||
single_sheet_in_book = len(keys) == 1
|
single_sheet_in_book = len(keys) == 1
|
||||||
|
|
||||||
no_file_type = isstream(afile) and file_type is None
|
no_file_type = isstream(afile) and file_type is None
|
||||||
if no_file_type:
|
if no_file_type:
|
||||||
file_type = constants.FILE_FORMAT_CSV
|
file_type = constants.FILE_FORMAT_CSV
|
||||||
|
|
||||||
store_data(
|
|
||||||
afile,
|
|
||||||
to_store,
|
|
||||||
file_type=file_type,
|
|
||||||
single_sheet_in_book=single_sheet_in_book,
|
|
||||||
**keywords
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def store_data(afile, data, file_type=None, **keywords):
|
|
||||||
"""Non public function to store data to afile
|
|
||||||
|
|
||||||
:param filename: actual file name, a file stream or actual content
|
|
||||||
:param data: the data to be written
|
|
||||||
:param file_type: used only when filename is not a physial file name
|
|
||||||
:param keywords: any other parameters
|
|
||||||
"""
|
|
||||||
if isstream(afile):
|
if isstream(afile):
|
||||||
keywords.update(dict(file_stream=afile, file_type=file_type))
|
keywords.update(dict(file_stream=afile, file_type=file_type))
|
||||||
else:
|
else:
|
||||||
keywords.update(dict(file_name=afile, file_type=file_type))
|
keywords.update(dict(file_name=afile, file_type=file_type))
|
||||||
|
keywords["single_sheet_in_book"] = single_sheet_in_book
|
||||||
with get_writer(**keywords) as writer:
|
with get_writer(**keywords) as writer:
|
||||||
writer.write(data)
|
writer.write(to_store)
|
||||||
|
|
||||||
|
|
||||||
def load_data(
|
def load_data(
|
||||||
|
@ -180,34 +179,61 @@ def load_data(
|
||||||
try:
|
try:
|
||||||
file_type = file_name.split(".")[-1]
|
file_type = file_name.split(".")[-1]
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
raise Exception("file_name should be a string type")
|
raise Exception(constants.MESSAGE_FILE_NAME_SHOULD_BE_STRING)
|
||||||
|
|
||||||
reader = READERS.get_a_plugin(file_type, library)
|
try:
|
||||||
if file_name:
|
reader = OLD_READERS.get_a_plugin(file_type, library)
|
||||||
reader.open(file_name, **keywords)
|
except (NoSupportingPluginFound, SupportingPluginAvailableButNotInstalled):
|
||||||
elif file_content:
|
reader = Reader(file_type, library)
|
||||||
reader.open_content(file_content, **keywords)
|
|
||||||
elif file_stream:
|
|
||||||
reader.open_stream(file_stream, **keywords)
|
|
||||||
if sheet_name:
|
|
||||||
result = reader.read_sheet_by_name(sheet_name)
|
|
||||||
elif sheet_index is not None:
|
|
||||||
result = reader.read_sheet_by_index(sheet_index)
|
|
||||||
elif sheets is not None:
|
|
||||||
result = reader.read_many(sheets)
|
|
||||||
else:
|
|
||||||
result = reader.read_all()
|
|
||||||
if streaming is False:
|
|
||||||
for key in result.keys():
|
|
||||||
result[key] = list(result[key])
|
|
||||||
reader.close()
|
|
||||||
reader = None
|
|
||||||
|
|
||||||
return result, reader
|
try:
|
||||||
|
if file_name:
|
||||||
|
reader.open(file_name, **keywords)
|
||||||
|
elif file_content:
|
||||||
|
reader.open_content(file_content, **keywords)
|
||||||
|
elif file_stream:
|
||||||
|
reader.open_stream(file_stream, **keywords)
|
||||||
|
else:
|
||||||
|
raise IOError("Unrecognized options")
|
||||||
|
if sheet_name:
|
||||||
|
result = reader.read_sheet_by_name(sheet_name)
|
||||||
|
elif sheet_index is not None:
|
||||||
|
result = reader.read_sheet_by_index(sheet_index)
|
||||||
|
elif sheets is not None:
|
||||||
|
result = reader.read_many(sheets)
|
||||||
|
else:
|
||||||
|
result = reader.read_all()
|
||||||
|
if streaming is False:
|
||||||
|
for key in result.keys():
|
||||||
|
result[key] = list(result[key])
|
||||||
|
reader.close()
|
||||||
|
reader = None
|
||||||
|
|
||||||
|
return result, reader
|
||||||
|
except NoSupportingPluginFound:
|
||||||
|
if file_name:
|
||||||
|
if os.path.exists(file_name):
|
||||||
|
if os.path.isfile(file_name):
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
raise IOError(
|
||||||
|
constants.MESSAGE_NOT_FILE_FORMATTER % file_name
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise IOError(
|
||||||
|
constants.MESSAGE_FILE_DOES_NOT_EXIST % file_name
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
def get_writer(
|
def get_writer(
|
||||||
file_name=None, file_stream=None, file_type=None, library=None, **keywords
|
file_name=None,
|
||||||
|
file_stream=None,
|
||||||
|
file_type=None,
|
||||||
|
library=None,
|
||||||
|
force_file_type=None,
|
||||||
|
**keywords
|
||||||
):
|
):
|
||||||
"""find a suitable writer"""
|
"""find a suitable writer"""
|
||||||
inputs = [file_name, file_stream]
|
inputs = [file_name, file_stream]
|
||||||
|
@ -217,15 +243,23 @@ def get_writer(
|
||||||
raise IOError(constants.MESSAGE_ERROR_02)
|
raise IOError(constants.MESSAGE_ERROR_02)
|
||||||
|
|
||||||
file_type_given = True
|
file_type_given = True
|
||||||
|
|
||||||
if file_type is None and file_name:
|
if file_type is None and file_name:
|
||||||
try:
|
if force_file_type:
|
||||||
file_type = file_name.split(".")[-1]
|
file_type = force_file_type
|
||||||
except AttributeError:
|
else:
|
||||||
raise Exception("file_name should be a string type")
|
try:
|
||||||
|
file_type = file_name.split(".")[-1]
|
||||||
|
except AttributeError:
|
||||||
|
raise Exception(constants.MESSAGE_FILE_NAME_SHOULD_BE_STRING)
|
||||||
|
|
||||||
file_type_given = False
|
file_type_given = False
|
||||||
|
|
||||||
writer = WRITERS.get_a_plugin(file_type, library)
|
try:
|
||||||
|
writer = OLD_WRITERS.get_a_plugin(file_type, library)
|
||||||
|
except (NoSupportingPluginFound, SupportingPluginAvailableButNotInstalled):
|
||||||
|
writer = Writer(file_type, library)
|
||||||
|
|
||||||
if file_name:
|
if file_name:
|
||||||
if file_type_given:
|
if file_type_given:
|
||||||
writer.open_content(file_name, **keywords)
|
writer.open_content(file_name, **keywords)
|
||||||
|
@ -235,3 +269,7 @@ def get_writer(
|
||||||
writer.open_stream(file_stream, **keywords)
|
writer.open_stream(file_stream, **keywords)
|
||||||
# else: is resolved by earlier raise statement
|
# else: is resolved by earlier raise statement
|
||||||
return writer
|
return writer
|
||||||
|
|
||||||
|
|
||||||
|
# backward compactibility
|
||||||
|
store_data = save_data
|
||||||
|
|
|
@ -4,11 +4,10 @@
|
||||||
|
|
||||||
Control file streams
|
Control file streams
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io._compact import StringIO, BytesIO
|
from pyexcel_io._compact import BytesIO, StringIO
|
||||||
|
|
||||||
|
|
||||||
MIME_TYPES = {}
|
MIME_TYPES = {}
|
||||||
FILE_TYPES = ()
|
FILE_TYPES = ()
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
from .abstract_sheet import ISheet, ISheetWriter, NamedContent # noqa: F401
|
||||||
|
from .abstract_reader import IReader # noqa: F401
|
||||||
|
from .abstract_writer import IWriter # noqa: F401
|
|
@ -0,0 +1,21 @@
|
||||||
|
from .abstract_sheet import ISheet
|
||||||
|
|
||||||
|
|
||||||
|
class IReader(object):
|
||||||
|
"""
|
||||||
|
content_array should be a list of NamedContent
|
||||||
|
where: name is the sheet name,
|
||||||
|
payload is the native sheet.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def read_sheet(self, sheet_index) -> ISheet:
|
||||||
|
raise NotImplementedError("Read the sheet by index")
|
||||||
|
|
||||||
|
def sheet_names(self):
|
||||||
|
return [content.name for content in self.content_array]
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.content_array)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
raise NotImplementedError("Close the file")
|
|
@ -0,0 +1,31 @@
|
||||||
|
class ISheet(object):
|
||||||
|
def row_iterator(self):
|
||||||
|
raise NotImplementedError("iterate each row")
|
||||||
|
|
||||||
|
def column_iterator(self, row):
|
||||||
|
raise NotImplementedError("iterate each column at a given row")
|
||||||
|
|
||||||
|
|
||||||
|
class ISheetWriter(object):
|
||||||
|
def write_row(self, data_row):
|
||||||
|
raise NotImplementedError("How does your sheet write a row of data")
|
||||||
|
|
||||||
|
def write_array(self, table):
|
||||||
|
"""
|
||||||
|
For standalone usage, write an array
|
||||||
|
"""
|
||||||
|
for row in table:
|
||||||
|
self.write_row(row)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
raise NotImplementedError("How would you close your file")
|
||||||
|
|
||||||
|
|
||||||
|
class NamedContent(object):
|
||||||
|
"""
|
||||||
|
Helper class for content that does not have a name
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, name, payload):
|
||||||
|
self.name = name
|
||||||
|
self.payload = payload
|
|
@ -0,0 +1,15 @@
|
||||||
|
from .abstract_sheet import ISheetWriter
|
||||||
|
|
||||||
|
|
||||||
|
class IWriter(object):
|
||||||
|
def create_sheet(self, sheet_name) -> ISheetWriter:
|
||||||
|
raise NotImplementedError("Please implement a native sheet writer")
|
||||||
|
|
||||||
|
def write(self, incoming_dict):
|
||||||
|
for sheet_name in incoming_dict:
|
||||||
|
sheet_writer = self.create_sheet(sheet_name)
|
||||||
|
if sheet_writer:
|
||||||
|
sheet_writer.write_array(incoming_dict[sheet_name])
|
||||||
|
sheet_writer.close()
|
||||||
|
else:
|
||||||
|
raise Exception("Cannot create a sheet writer!")
|
|
@ -4,24 +4,23 @@
|
||||||
|
|
||||||
factory for getting readers and writers
|
factory for getting readers and writers
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from lml.loader import scan_plugins_regex
|
|
||||||
from lml.plugin import PluginManager
|
|
||||||
from lml.plugin import PluginInfoChain, PluginInfo
|
|
||||||
|
|
||||||
import pyexcel_io.utils as ioutils
|
import pyexcel_io.utils as ioutils
|
||||||
import pyexcel_io.manager as manager
|
import pyexcel_io.manager as manager
|
||||||
import pyexcel_io.exceptions as exceptions
|
|
||||||
import pyexcel_io.constants as constants
|
import pyexcel_io.constants as constants
|
||||||
|
import pyexcel_io.exceptions as exceptions
|
||||||
|
from lml.loader import scan_plugins_regex
|
||||||
|
from lml.plugin import PluginInfo, PluginManager, PluginInfoChain
|
||||||
|
|
||||||
ERROR_MESSAGE_FORMATTER = "one of these plugins for %s data in '%s': %s"
|
ERROR_MESSAGE_FORMATTER = "one of these plugins for %s data in '%s': %s"
|
||||||
UPGRADE_MESSAGE = "Please upgrade the plugin '%s' according to \
|
UPGRADE_MESSAGE = "Please upgrade the plugin '%s' according to \
|
||||||
plugin compactibility table."
|
plugin compactibility table."
|
||||||
READER_PLUGIN = "pyexcel-io reader"
|
READER_PLUGIN = "pyexcel-io reader"
|
||||||
|
READER_PLUGIN_V2 = "pyexcel-io v2 reader"
|
||||||
WRITER_PLUGIN = "pyexcel-io writer"
|
WRITER_PLUGIN = "pyexcel-io writer"
|
||||||
|
WRITER_PLUGIN_V2 = "pyexcel-io v2 writer"
|
||||||
|
|
||||||
|
|
||||||
class IOPluginInfo(PluginInfo):
|
class IOPluginInfo(PluginInfo):
|
||||||
|
@ -33,7 +32,7 @@ class IOPluginInfo(PluginInfo):
|
||||||
|
|
||||||
|
|
||||||
class IOPluginInfoChain(PluginInfoChain):
|
class IOPluginInfoChain(PluginInfoChain):
|
||||||
"""provide custom functions to add a reader and a writer """
|
"""provide custom functions to add a reader and a writer"""
|
||||||
|
|
||||||
def add_a_reader(
|
def add_a_reader(
|
||||||
self,
|
self,
|
||||||
|
@ -41,7 +40,7 @@ class IOPluginInfoChain(PluginInfoChain):
|
||||||
file_types=None,
|
file_types=None,
|
||||||
stream_type=None,
|
stream_type=None,
|
||||||
):
|
):
|
||||||
""" add pyexcle-io reader plugin info """
|
"""add pyexcle-io reader plugin info"""
|
||||||
a_plugin_info = IOPluginInfo(
|
a_plugin_info = IOPluginInfo(
|
||||||
READER_PLUGIN,
|
READER_PLUGIN,
|
||||||
self._get_abs_path(relative_plugin_class_path),
|
self._get_abs_path(relative_plugin_class_path),
|
||||||
|
@ -56,7 +55,7 @@ class IOPluginInfoChain(PluginInfoChain):
|
||||||
file_types=None,
|
file_types=None,
|
||||||
stream_type=None,
|
stream_type=None,
|
||||||
):
|
):
|
||||||
""" add pyexcle-io writer plugin info """
|
"""add pyexcle-io writer plugin info"""
|
||||||
a_plugin_info = IOPluginInfo(
|
a_plugin_info = IOPluginInfo(
|
||||||
WRITER_PLUGIN,
|
WRITER_PLUGIN,
|
||||||
self._get_abs_path(relative_plugin_class_path),
|
self._get_abs_path(relative_plugin_class_path),
|
||||||
|
@ -66,6 +65,50 @@ class IOPluginInfoChain(PluginInfoChain):
|
||||||
return self.add_a_plugin_instance(a_plugin_info)
|
return self.add_a_plugin_instance(a_plugin_info)
|
||||||
|
|
||||||
|
|
||||||
|
class IOPluginInfoChainV2(PluginInfoChain):
|
||||||
|
"""provide custom functions to add a reader and a writer"""
|
||||||
|
|
||||||
|
def add_a_reader(
|
||||||
|
self,
|
||||||
|
relative_plugin_class_path=None,
|
||||||
|
locations=(),
|
||||||
|
file_types=None,
|
||||||
|
stream_type=None,
|
||||||
|
):
|
||||||
|
"""add pyexcle-io reader plugin info"""
|
||||||
|
a_plugin_info = IOPluginInfo(
|
||||||
|
READER_PLUGIN_V2,
|
||||||
|
self._get_abs_path(relative_plugin_class_path),
|
||||||
|
file_types=[
|
||||||
|
f"{location}-{file_type}"
|
||||||
|
for file_type in file_types
|
||||||
|
for location in locations
|
||||||
|
],
|
||||||
|
stream_type=stream_type,
|
||||||
|
)
|
||||||
|
return self.add_a_plugin_instance(a_plugin_info)
|
||||||
|
|
||||||
|
def add_a_writer(
|
||||||
|
self,
|
||||||
|
relative_plugin_class_path=None,
|
||||||
|
locations=(),
|
||||||
|
file_types=(),
|
||||||
|
stream_type=None,
|
||||||
|
):
|
||||||
|
"""add pyexcle-io writer plugin info"""
|
||||||
|
a_plugin_info = IOPluginInfo(
|
||||||
|
WRITER_PLUGIN_V2,
|
||||||
|
self._get_abs_path(relative_plugin_class_path),
|
||||||
|
file_types=[
|
||||||
|
f"{location}-{file_type}"
|
||||||
|
for file_type in file_types
|
||||||
|
for location in locations
|
||||||
|
],
|
||||||
|
stream_type=stream_type,
|
||||||
|
)
|
||||||
|
return self.add_a_plugin_instance(a_plugin_info)
|
||||||
|
|
||||||
|
|
||||||
class IOManager(PluginManager):
|
class IOManager(PluginManager):
|
||||||
"""Manage pyexcel-io plugins"""
|
"""Manage pyexcel-io plugins"""
|
||||||
|
|
||||||
|
@ -81,13 +124,16 @@ class IOManager(PluginManager):
|
||||||
_do_additional_registration(plugin_info)
|
_do_additional_registration(plugin_info)
|
||||||
|
|
||||||
def register_a_plugin(self, cls, plugin_info):
|
def register_a_plugin(self, cls, plugin_info):
|
||||||
""" for dynamically loaded plugin """
|
"""for dynamically loaded plugin"""
|
||||||
PluginManager.register_a_plugin(self, cls, plugin_info)
|
PluginManager.register_a_plugin(self, cls, plugin_info)
|
||||||
_do_additional_registration(plugin_info)
|
_do_additional_registration(plugin_info)
|
||||||
|
|
||||||
def get_a_plugin(self, file_type=None, library=None, **keywords):
|
def get_a_plugin(self, file_type=None, library=None, **keywords):
|
||||||
__file_type = file_type.lower()
|
__file_type = file_type.lower()
|
||||||
plugin = self.load_me_now(__file_type, library=library)
|
try:
|
||||||
|
plugin = self.load_me_now(__file_type, library=library)
|
||||||
|
except Exception:
|
||||||
|
self.raise_exception(__file_type)
|
||||||
handler = plugin()
|
handler = plugin()
|
||||||
handler.set_type(__file_type)
|
handler.set_type(__file_type)
|
||||||
return handler
|
return handler
|
||||||
|
@ -112,7 +158,7 @@ class IOManager(PluginManager):
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_all_formats(self):
|
def get_all_formats(self):
|
||||||
""" return all supported formats """
|
"""return all supported formats"""
|
||||||
all_formats = set(
|
all_formats = set(
|
||||||
list(self.registry.keys()) + list(self.known_plugins.keys())
|
list(self.registry.keys()) + list(self.known_plugins.keys())
|
||||||
)
|
)
|
||||||
|
@ -122,14 +168,88 @@ class IOManager(PluginManager):
|
||||||
return all_formats
|
return all_formats
|
||||||
|
|
||||||
|
|
||||||
|
class NewIOManager(IOManager):
|
||||||
|
def load_me_later(self, plugin_info):
|
||||||
|
PluginManager.load_me_later(self, plugin_info)
|
||||||
|
_do_additional_registration_for_new_plugins(plugin_info)
|
||||||
|
|
||||||
|
def register_a_plugin(self, cls, plugin_info):
|
||||||
|
"""for dynamically loaded plugin"""
|
||||||
|
PluginManager.register_a_plugin(self, cls, plugin_info)
|
||||||
|
_do_additional_registration_for_new_plugins(plugin_info)
|
||||||
|
|
||||||
|
def get_a_plugin(
|
||||||
|
self, file_type=None, location=None, library=None, **keywords
|
||||||
|
):
|
||||||
|
__file_type = file_type.lower()
|
||||||
|
plugin = self.load_me_now(f"{location}-{__file_type}", library=library)
|
||||||
|
return plugin
|
||||||
|
|
||||||
|
def raise_exception(self, file_type):
|
||||||
|
file_type = file_type.split("-")[1]
|
||||||
|
plugins = self.known_plugins.get(file_type, None)
|
||||||
|
if plugins:
|
||||||
|
message = "Please install "
|
||||||
|
if len(plugins) > 1:
|
||||||
|
message += ERROR_MESSAGE_FORMATTER % (
|
||||||
|
self.action,
|
||||||
|
file_type,
|
||||||
|
",".join(plugins),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
message += plugins[0]
|
||||||
|
raise exceptions.SupportingPluginAvailableButNotInstalled(message)
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise exceptions.NoSupportingPluginFound(
|
||||||
|
"No suitable library found for %s" % file_type
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_all_formats(self):
|
||||||
|
"""return all supported formats"""
|
||||||
|
all_formats = set(
|
||||||
|
[x.split("-")[1] for x in self.registry.keys()]
|
||||||
|
+ list(self.known_plugins.keys())
|
||||||
|
)
|
||||||
|
return all_formats
|
||||||
|
|
||||||
|
|
||||||
def _do_additional_registration(plugin_info):
|
def _do_additional_registration(plugin_info):
|
||||||
for file_type in plugin_info.tags():
|
for file_type in plugin_info.tags():
|
||||||
manager.register_stream_type(file_type, plugin_info.stream_type)
|
manager.register_stream_type(file_type, plugin_info.stream_type)
|
||||||
manager.register_a_file_type(file_type, plugin_info.stream_type, None)
|
manager.register_a_file_type(file_type, plugin_info.stream_type, None)
|
||||||
|
|
||||||
|
|
||||||
READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS)
|
def _do_additional_registration_for_new_plugins(plugin_info):
|
||||||
WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS)
|
for file_type in plugin_info.tags():
|
||||||
|
manager.register_stream_type(
|
||||||
|
file_type.split("-")[1], plugin_info.stream_type
|
||||||
|
)
|
||||||
|
manager.register_a_file_type(
|
||||||
|
file_type.split("-")[1], plugin_info.stream_type, None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AllReaders:
|
||||||
|
def get_all_formats(self):
|
||||||
|
return OLD_READERS.get_all_formats().union(
|
||||||
|
NEW_READERS.get_all_formats()
|
||||||
|
) - set([constants.DB_SQL, constants.DB_DJANGO])
|
||||||
|
|
||||||
|
|
||||||
|
class AllWriters:
|
||||||
|
def get_all_formats(self):
|
||||||
|
return OLD_WRITERS.get_all_formats().union(
|
||||||
|
NEW_WRITERS.get_all_formats()
|
||||||
|
) - set([constants.DB_SQL, constants.DB_DJANGO])
|
||||||
|
|
||||||
|
|
||||||
|
OLD_READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS)
|
||||||
|
OLD_WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS)
|
||||||
|
NEW_WRITERS = NewIOManager(WRITER_PLUGIN_V2, ioutils.AVAILABLE_WRITERS)
|
||||||
|
NEW_READERS = NewIOManager(READER_PLUGIN_V2, ioutils.AVAILABLE_READERS)
|
||||||
|
READERS = AllReaders()
|
||||||
|
WRITERS = AllWriters()
|
||||||
|
|
||||||
|
|
||||||
def load_plugins(plugin_name_patterns, path, black_list, white_list):
|
def load_plugins(plugin_name_patterns, path, black_list, white_list):
|
||||||
|
@ -138,5 +258,5 @@ def load_plugins(plugin_name_patterns, path, black_list, white_list):
|
||||||
plugin_name_patterns=plugin_name_patterns,
|
plugin_name_patterns=plugin_name_patterns,
|
||||||
pyinstaller_path=path,
|
pyinstaller_path=path,
|
||||||
black_list=black_list,
|
black_list=black_list,
|
||||||
white_list=white_list
|
white_list=white_list,
|
||||||
)
|
)
|
||||||
|
|
|
@ -0,0 +1,126 @@
|
||||||
|
from pyexcel_io.sheet import SheetReader
|
||||||
|
from pyexcel_io.plugins import NEW_READERS
|
||||||
|
from pyexcel_io._compact import OrderedDict
|
||||||
|
|
||||||
|
|
||||||
|
def clean_keywords(keywords):
|
||||||
|
sheet_keywords = {}
|
||||||
|
native_sheet_keywords = {}
|
||||||
|
args_list = [
|
||||||
|
"start_row",
|
||||||
|
"row_limit",
|
||||||
|
"start_column",
|
||||||
|
"column_limit",
|
||||||
|
"skip_column_func",
|
||||||
|
"skip_row_func",
|
||||||
|
"skip_empty_rows",
|
||||||
|
"row_renderer",
|
||||||
|
"keep_trailing_empty_cells",
|
||||||
|
]
|
||||||
|
for arg in keywords:
|
||||||
|
if arg in args_list:
|
||||||
|
sheet_keywords[arg] = keywords[arg]
|
||||||
|
else:
|
||||||
|
native_sheet_keywords[arg] = keywords[arg]
|
||||||
|
return sheet_keywords, native_sheet_keywords
|
||||||
|
|
||||||
|
|
||||||
|
class Reader(object):
|
||||||
|
def __init__(self, file_type, library=None):
|
||||||
|
self.file_type = file_type
|
||||||
|
self.library = library
|
||||||
|
self.keywords = None
|
||||||
|
|
||||||
|
# if you know which reader class to use, this attribute allows
|
||||||
|
# you to set reader class externally. Since there is no
|
||||||
|
# so call private field in Python, I am not going to create
|
||||||
|
# useless setter and getter functions like Java.
|
||||||
|
# in pyexcel, this attribute is mainly used for testing
|
||||||
|
self.reader_class = None
|
||||||
|
|
||||||
|
def open(self, file_name, **keywords):
|
||||||
|
if self.reader_class is None:
|
||||||
|
self.reader_class = NEW_READERS.get_a_plugin(
|
||||||
|
self.file_type, location="file", library=self.library
|
||||||
|
)
|
||||||
|
self.keywords, native_sheet_keywords = clean_keywords(keywords)
|
||||||
|
self.reader = self.reader_class(
|
||||||
|
file_name, self.file_type, **native_sheet_keywords
|
||||||
|
)
|
||||||
|
return self.reader
|
||||||
|
|
||||||
|
def open_content(self, file_content, **keywords):
|
||||||
|
self.keywords, native_sheet_keywords = clean_keywords(keywords)
|
||||||
|
if self.reader_class is None:
|
||||||
|
self.reader_class = NEW_READERS.get_a_plugin(
|
||||||
|
self.file_type, location="content", library=self.library
|
||||||
|
)
|
||||||
|
self.reader = self.reader_class(
|
||||||
|
file_content, self.file_type, **native_sheet_keywords
|
||||||
|
)
|
||||||
|
return self.reader
|
||||||
|
|
||||||
|
def open_stream(self, file_stream, **keywords):
|
||||||
|
self.keywords, native_sheet_keywords = clean_keywords(keywords)
|
||||||
|
if self.reader_class is None:
|
||||||
|
self.reader_class = NEW_READERS.get_a_plugin(
|
||||||
|
self.file_type, location="memory", library=self.library
|
||||||
|
)
|
||||||
|
self.reader = self.reader_class(
|
||||||
|
file_stream, self.file_type, **native_sheet_keywords
|
||||||
|
)
|
||||||
|
return self.reader
|
||||||
|
|
||||||
|
def read_sheet_by_name(self, sheet_name):
|
||||||
|
"""
|
||||||
|
read a named sheet from a excel data book
|
||||||
|
"""
|
||||||
|
sheet_names = self.reader.sheet_names()
|
||||||
|
index = sheet_names.index(sheet_name)
|
||||||
|
|
||||||
|
return self.read_sheet_by_index(index)
|
||||||
|
|
||||||
|
def read_sheet_by_index(self, sheet_index):
|
||||||
|
sheet_reader = self.reader.read_sheet(sheet_index)
|
||||||
|
sheet_names = self.reader.sheet_names()
|
||||||
|
sheet = EncapsulatedSheetReader(sheet_reader, **self.keywords)
|
||||||
|
return {sheet_names[sheet_index]: sheet.to_array()}
|
||||||
|
|
||||||
|
def read_all(self):
|
||||||
|
"""
|
||||||
|
read everything from a excel data book
|
||||||
|
"""
|
||||||
|
result = OrderedDict()
|
||||||
|
for sheet_index in range(len(self.reader)):
|
||||||
|
content_dict = self.read_sheet_by_index(sheet_index)
|
||||||
|
result.update(content_dict)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def read_many(self, sheets):
|
||||||
|
"""
|
||||||
|
read everything from a excel data book
|
||||||
|
"""
|
||||||
|
result = OrderedDict()
|
||||||
|
for sheet in sheets:
|
||||||
|
if isinstance(sheet, int):
|
||||||
|
result.update(self.read_sheet_by_index(sheet))
|
||||||
|
else:
|
||||||
|
result.update(self.read_sheet_by_name(sheet))
|
||||||
|
return result
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
return self.reader.close()
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, a_type, value, traceback):
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
|
||||||
|
class EncapsulatedSheetReader(SheetReader):
|
||||||
|
def row_iterator(self):
|
||||||
|
yield from self._native_sheet.row_iterator()
|
||||||
|
|
||||||
|
def column_iterator(self, row):
|
||||||
|
yield from self._native_sheet.column_iterator(row)
|
|
@ -4,26 +4,34 @@
|
||||||
|
|
||||||
file readers
|
file readers
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io.plugins import IOPluginInfoChain
|
from pyexcel_io.plugins import IOPluginInfoChainV2
|
||||||
|
|
||||||
|
IOPluginInfoChainV2(__name__).add_a_reader(
|
||||||
IOPluginInfoChain(__name__).add_a_reader(
|
relative_plugin_class_path="csv_in_file.FileReader",
|
||||||
relative_plugin_class_path="csvr.CSVBookReader",
|
locations=["file"],
|
||||||
file_types=["csv"],
|
file_types=["csv", "tsv"],
|
||||||
stream_type="text",
|
stream_type="text",
|
||||||
).add_a_reader(
|
).add_a_reader(
|
||||||
relative_plugin_class_path="tsv.TSVBookReader",
|
relative_plugin_class_path="csv_content.ContentReader",
|
||||||
file_types=["tsv"],
|
locations=["content"],
|
||||||
|
file_types=["csv", "tsv"],
|
||||||
stream_type="text",
|
stream_type="text",
|
||||||
).add_a_reader(
|
).add_a_reader(
|
||||||
relative_plugin_class_path="csvz.CSVZipBookReader",
|
relative_plugin_class_path="csv_in_memory.MemoryReader",
|
||||||
file_types=["csvz"],
|
locations=["memory"],
|
||||||
|
file_types=["csv", "tsv"],
|
||||||
|
stream_type="text",
|
||||||
|
).add_a_reader(
|
||||||
|
relative_plugin_class_path="csvz.FileReader",
|
||||||
|
file_types=["csvz", "tsvz"],
|
||||||
|
locations=["file", "memory"],
|
||||||
stream_type="binary",
|
stream_type="binary",
|
||||||
).add_a_reader(
|
).add_a_reader(
|
||||||
relative_plugin_class_path="tsvz.TSVZipBookReader",
|
relative_plugin_class_path="csvz.ContentReader",
|
||||||
file_types=["tsvz"],
|
file_types=["csvz", "tsvz"],
|
||||||
|
locations=["content"],
|
||||||
stream_type="binary",
|
stream_type="binary",
|
||||||
)
|
)
|
||||||
|
|
|
@ -0,0 +1,27 @@
|
||||||
|
import mmap
|
||||||
|
|
||||||
|
from pyexcel_io.book import _convert_content_to_stream
|
||||||
|
from pyexcel_io.readers.csv_sheet import CSVMemoryMapIterator
|
||||||
|
from pyexcel_io.readers.csv_in_memory import MemoryReader
|
||||||
|
|
||||||
|
|
||||||
|
class ContentReader(MemoryReader):
|
||||||
|
def __init__(self, file_content, file_type, **keywords):
|
||||||
|
file_stream = ContentReader.convert_content_to_stream(
|
||||||
|
file_content, file_type, **keywords
|
||||||
|
)
|
||||||
|
super().__init__(file_stream, file_type, **keywords)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def convert_content_to_stream(file_content, file_type, **keywords):
|
||||||
|
encoding = keywords.get("encoding", "utf-8")
|
||||||
|
if isinstance(file_content, mmap.mmap):
|
||||||
|
# load from mmap
|
||||||
|
file_stream = CSVMemoryMapIterator(file_content, encoding)
|
||||||
|
else:
|
||||||
|
if isinstance(file_content, bytes):
|
||||||
|
file_content = file_content.decode(encoding)
|
||||||
|
|
||||||
|
file_stream = _convert_content_to_stream(file_content, file_type)
|
||||||
|
|
||||||
|
return file_stream
|
|
@ -0,0 +1,64 @@
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import glob
|
||||||
|
|
||||||
|
from pyexcel_io import constants
|
||||||
|
from pyexcel_io.sheet import NamedContent
|
||||||
|
from pyexcel_io.plugin_api import IReader
|
||||||
|
from pyexcel_io.readers.csv_sheet import CSVFileReader
|
||||||
|
|
||||||
|
DEFAULT_NEWLINE = "\r\n"
|
||||||
|
|
||||||
|
|
||||||
|
class FileReader(IReader):
|
||||||
|
def __init__(self, file_name, file_type, **keywords):
|
||||||
|
"""Load content from a file
|
||||||
|
:params str filename: an accessible file path
|
||||||
|
:returns: a book
|
||||||
|
"""
|
||||||
|
self.handles = []
|
||||||
|
self.keywords = keywords
|
||||||
|
if file_type == constants.FILE_FORMAT_TSV:
|
||||||
|
self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
||||||
|
self.__line_terminator = keywords.get(
|
||||||
|
constants.KEYWORD_LINE_TERMINATOR, DEFAULT_NEWLINE
|
||||||
|
)
|
||||||
|
names = os.path.splitext(file_name)
|
||||||
|
filepattern = "%s%s*%s*%s" % (
|
||||||
|
names[0],
|
||||||
|
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
||||||
|
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
||||||
|
names[1],
|
||||||
|
)
|
||||||
|
filelist = glob.glob(filepattern)
|
||||||
|
if len(filelist) == 0:
|
||||||
|
file_parts = os.path.split(file_name)
|
||||||
|
self.content_array = [NamedContent(file_parts[-1], file_name)]
|
||||||
|
|
||||||
|
else:
|
||||||
|
matcher = "%s%s(.*)%s(.*)%s" % (
|
||||||
|
names[0],
|
||||||
|
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
||||||
|
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
||||||
|
names[1],
|
||||||
|
)
|
||||||
|
tmp_file_list = []
|
||||||
|
for filen in filelist:
|
||||||
|
result = re.match(matcher, filen)
|
||||||
|
tmp_file_list.append((result.group(1), result.group(2), filen))
|
||||||
|
ret = []
|
||||||
|
for lsheetname, index, filen in sorted(
|
||||||
|
tmp_file_list, key=lambda row: row[1]
|
||||||
|
):
|
||||||
|
ret.append(NamedContent(lsheetname, filen))
|
||||||
|
self.content_array = ret
|
||||||
|
|
||||||
|
def read_sheet(self, index):
|
||||||
|
reader = CSVFileReader(self.content_array[index], **self.keywords)
|
||||||
|
self.handles.append(reader)
|
||||||
|
return reader
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
for reader in self.handles:
|
||||||
|
reader.close()
|
||||||
|
self.handles = []
|
|
@ -0,0 +1,62 @@
|
||||||
|
import re
|
||||||
|
|
||||||
|
import pyexcel_io._compact as compact
|
||||||
|
from pyexcel_io import constants
|
||||||
|
from pyexcel_io.sheet import NamedContent
|
||||||
|
from pyexcel_io.plugin_api import IReader
|
||||||
|
from pyexcel_io.readers.csv_sheet import CSVinMemoryReader
|
||||||
|
|
||||||
|
DEFAULT_SHEET_SEPARATOR_FORMATTER = f"---{constants.DEFAULT_NAME}---%s"
|
||||||
|
|
||||||
|
|
||||||
|
class MemoryReader(IReader):
|
||||||
|
def __init__(
|
||||||
|
self, file_stream, file_type, multiple_sheets=False, **keywords
|
||||||
|
):
|
||||||
|
"""Load content from memory
|
||||||
|
:params stream file_content: the actual file content in memory
|
||||||
|
:returns: a book
|
||||||
|
"""
|
||||||
|
self.handles = []
|
||||||
|
self.keywords = keywords
|
||||||
|
if file_type == constants.FILE_FORMAT_TSV:
|
||||||
|
self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
||||||
|
self.file_type = file_type
|
||||||
|
|
||||||
|
self.__load_from_memory_flag = True
|
||||||
|
self.__line_terminator = keywords.get(
|
||||||
|
constants.KEYWORD_LINE_TERMINATOR, constants.DEFAULT_CSV_NEWLINE
|
||||||
|
)
|
||||||
|
separator = DEFAULT_SHEET_SEPARATOR_FORMATTER % self.__line_terminator
|
||||||
|
if multiple_sheets:
|
||||||
|
# will be slow for large files
|
||||||
|
file_stream.seek(0)
|
||||||
|
content = file_stream.read()
|
||||||
|
sheets = content.split(separator)
|
||||||
|
named_contents = []
|
||||||
|
for sheet in sheets:
|
||||||
|
if sheet == "": # skip empty named sheet
|
||||||
|
continue
|
||||||
|
|
||||||
|
lines = sheet.split(self.__line_terminator)
|
||||||
|
result = re.match(constants.SEPARATOR_MATCHER, lines[0])
|
||||||
|
new_content = "\n".join(lines[1:])
|
||||||
|
new_sheet = NamedContent(
|
||||||
|
result.group(1), compact.StringIO(new_content)
|
||||||
|
)
|
||||||
|
named_contents.append(new_sheet)
|
||||||
|
self.content_array = named_contents
|
||||||
|
|
||||||
|
else:
|
||||||
|
if hasattr(file_stream, "seek"):
|
||||||
|
file_stream.seek(0)
|
||||||
|
self.content_array = [NamedContent(self.file_type, file_stream)]
|
||||||
|
|
||||||
|
def read_sheet(self, index):
|
||||||
|
reader = CSVinMemoryReader(self.content_array[index], **self.keywords)
|
||||||
|
self.handles.append(reader)
|
||||||
|
return reader
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
for reader in self.handles:
|
||||||
|
reader.close()
|
|
@ -0,0 +1,192 @@
|
||||||
|
"""
|
||||||
|
pyexcel_io.readers.csv_sheet
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
csv file reader
|
||||||
|
|
||||||
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
|
:license: New BSD License, see LICENSE for more details
|
||||||
|
"""
|
||||||
|
import csv
|
||||||
|
|
||||||
|
import pyexcel_io.service as service
|
||||||
|
import pyexcel_io._compact as compact
|
||||||
|
import pyexcel_io.constants as constants
|
||||||
|
from pyexcel_io.plugin_api import ISheet
|
||||||
|
|
||||||
|
DEFAULT_SEPARATOR = "__"
|
||||||
|
DEFAULT_SHEET_SEPARATOR_FORMATTER = "---%s---" % constants.DEFAULT_NAME + "%s"
|
||||||
|
SEPARATOR_MATCHER = "---%s:(.*)---" % constants.DEFAULT_NAME
|
||||||
|
DEFAULT_CSV_STREAM_FILE_FORMATTER = (
|
||||||
|
"---%s:" % constants.DEFAULT_NAME + "%s---%s"
|
||||||
|
)
|
||||||
|
DEFAULT_NEWLINE = "\r\n"
|
||||||
|
BOM_LITTLE_ENDIAN = b"\xff\xfe"
|
||||||
|
BOM_BIG_ENDIAN = b"\xfe\ff"
|
||||||
|
LITTLE_ENDIAN = 0
|
||||||
|
BIG_ENDIAN = 1
|
||||||
|
|
||||||
|
|
||||||
|
class CSVMemoryMapIterator(object):
|
||||||
|
"""
|
||||||
|
Wrapper class for mmap object
|
||||||
|
|
||||||
|
mmap object does not handle encoding at all. This class
|
||||||
|
provide the necessary transcoding for utf-8, utf-16 and utf-32
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, mmap_obj, encoding):
|
||||||
|
self.__mmap_obj = mmap_obj
|
||||||
|
self.__encoding = encoding
|
||||||
|
self.__count = 0
|
||||||
|
self.__endian = LITTLE_ENDIAN
|
||||||
|
if encoding == "utf-8":
|
||||||
|
# ..\r\x00\n
|
||||||
|
# \x00\x..
|
||||||
|
self.__zeros_left_in_2_row = 0
|
||||||
|
elif encoding == "utf-16":
|
||||||
|
# ..\r\x00\n
|
||||||
|
# \x00\x..
|
||||||
|
self.__zeros_left_in_2_row = 1
|
||||||
|
elif encoding == "utf-32":
|
||||||
|
# \r\x00\x00\x00\n
|
||||||
|
# \x00\x00\x00\x..
|
||||||
|
self.__zeros_left_in_2_row = 3
|
||||||
|
elif encoding in ["utf-32-be", "utf-16-be"]:
|
||||||
|
self.__zeros_left_in_2_row = 0
|
||||||
|
self.__endian = BIG_ENDIAN
|
||||||
|
elif encoding == "utf-32-le":
|
||||||
|
self.__zeros_left_in_2_row = 3
|
||||||
|
self.__endian = LITTLE_ENDIAN
|
||||||
|
elif encoding == "utf-16-le":
|
||||||
|
self.__zeros_left_in_2_row = 1
|
||||||
|
self.__endian = LITTLE_ENDIAN
|
||||||
|
else:
|
||||||
|
raise Exception("Encoding %s is not supported" % encoding)
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __next__(self):
|
||||||
|
line = self.__mmap_obj.readline()
|
||||||
|
if self.__count == 0:
|
||||||
|
utf_16_32 = (
|
||||||
|
self.__encoding == "utf-16" or self.__encoding == "utf-32"
|
||||||
|
)
|
||||||
|
if utf_16_32:
|
||||||
|
bom_header = line[:2]
|
||||||
|
if bom_header == BOM_BIG_ENDIAN:
|
||||||
|
self.__endian = BIG_ENDIAN
|
||||||
|
elif self.__endian == LITTLE_ENDIAN:
|
||||||
|
line = line[self.__zeros_left_in_2_row :] # noqa: E203
|
||||||
|
if self.__endian == LITTLE_ENDIAN:
|
||||||
|
line = line.rstrip()
|
||||||
|
line = line.decode(self.__encoding)
|
||||||
|
self.__count += 1
|
||||||
|
if line == "":
|
||||||
|
raise StopIteration
|
||||||
|
|
||||||
|
return line
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class CSVSheetReader(ISheet):
|
||||||
|
"""generic csv file reader"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
sheet,
|
||||||
|
encoding="utf-8",
|
||||||
|
auto_detect_float=True,
|
||||||
|
ignore_infinity=True,
|
||||||
|
auto_detect_int=True,
|
||||||
|
auto_detect_datetime=True,
|
||||||
|
pep_0515_off=True,
|
||||||
|
ignore_nan_text=False,
|
||||||
|
default_float_nan=None,
|
||||||
|
**keywords
|
||||||
|
):
|
||||||
|
self._native_sheet = sheet
|
||||||
|
self._encoding = encoding
|
||||||
|
self.__auto_detect_int = auto_detect_int
|
||||||
|
self.__auto_detect_float = auto_detect_float
|
||||||
|
self.__ignore_infinity = ignore_infinity
|
||||||
|
self.__auto_detect_datetime = auto_detect_datetime
|
||||||
|
self.__file_handle = None
|
||||||
|
self.__pep_0515_off = pep_0515_off
|
||||||
|
self.__ignore_nan_text = ignore_nan_text
|
||||||
|
self.__default_float_nan = default_float_nan
|
||||||
|
self._keywords = keywords
|
||||||
|
|
||||||
|
def get_file_handle(self):
|
||||||
|
"""return me unicde reader for csv"""
|
||||||
|
raise NotImplementedError("Please implement get_file_handle()")
|
||||||
|
|
||||||
|
def row_iterator(self):
|
||||||
|
self.__file_handle = self.get_file_handle()
|
||||||
|
return csv.reader(self.__file_handle, **self._keywords)
|
||||||
|
|
||||||
|
def column_iterator(self, row):
|
||||||
|
for element in row:
|
||||||
|
if element is not None and element != "":
|
||||||
|
element = self.__convert_cell(element)
|
||||||
|
yield element
|
||||||
|
|
||||||
|
def __convert_cell(self, csv_cell_text):
|
||||||
|
ret = None
|
||||||
|
if self.__auto_detect_int:
|
||||||
|
ret = service.detect_int_value(csv_cell_text, self.__pep_0515_off)
|
||||||
|
if ret is None and self.__auto_detect_float:
|
||||||
|
ret = service.detect_float_value(
|
||||||
|
csv_cell_text,
|
||||||
|
self.__pep_0515_off,
|
||||||
|
ignore_nan_text=self.__ignore_nan_text,
|
||||||
|
default_float_nan=self.__default_float_nan,
|
||||||
|
)
|
||||||
|
shall_we_ignore_the_conversion = (
|
||||||
|
ret in [float("inf"), float("-inf")]
|
||||||
|
) and self.__ignore_infinity
|
||||||
|
if shall_we_ignore_the_conversion:
|
||||||
|
ret = None
|
||||||
|
if ret is None and self.__auto_detect_datetime:
|
||||||
|
ret = service.detect_date_value(csv_cell_text)
|
||||||
|
if ret is None:
|
||||||
|
ret = csv_cell_text
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
if self.__file_handle:
|
||||||
|
self.__file_handle.close()
|
||||||
|
|
||||||
|
|
||||||
|
# else: means the generator has been run
|
||||||
|
# yes, no run, no file open.
|
||||||
|
|
||||||
|
|
||||||
|
class CSVFileReader(CSVSheetReader):
|
||||||
|
"""read csv from phyical file"""
|
||||||
|
|
||||||
|
def get_file_handle(self):
|
||||||
|
unicode_reader = open(
|
||||||
|
self._native_sheet.payload, "r", encoding=self._encoding
|
||||||
|
)
|
||||||
|
return unicode_reader
|
||||||
|
|
||||||
|
|
||||||
|
class CSVinMemoryReader(CSVSheetReader):
|
||||||
|
"""read csv file from memory"""
|
||||||
|
|
||||||
|
def get_file_handle(self):
|
||||||
|
if isinstance(self._native_sheet.payload, compact.BytesIO):
|
||||||
|
# please note that
|
||||||
|
# if the end developer feed us bytesio in python3
|
||||||
|
# we will do the conversion to StriongIO but that
|
||||||
|
# comes at a cost.
|
||||||
|
content = self._native_sheet.payload.read()
|
||||||
|
unicode_reader = compact.StringIO(content.decode(self._encoding))
|
||||||
|
else:
|
||||||
|
unicode_reader = self._native_sheet.payload
|
||||||
|
|
||||||
|
return unicode_reader
|
|
@ -1,369 +0,0 @@
|
||||||
"""
|
|
||||||
pyexcel_io.readers.csvr
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
csv file reader
|
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
|
||||||
:license: New BSD License, see LICENSE for more details
|
|
||||||
"""
|
|
||||||
import re
|
|
||||||
import os
|
|
||||||
import csv
|
|
||||||
import glob
|
|
||||||
import codecs
|
|
||||||
|
|
||||||
from pyexcel_io.book import BookReader
|
|
||||||
from pyexcel_io.sheet import SheetReader, NamedContent
|
|
||||||
import pyexcel_io._compact as compact
|
|
||||||
import pyexcel_io.constants as constants
|
|
||||||
import pyexcel_io.service as service
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_SEPARATOR = "__"
|
|
||||||
DEFAULT_SHEET_SEPARATOR_FORMATTER = "---%s---" % constants.DEFAULT_NAME + "%s"
|
|
||||||
SEPARATOR_MATCHER = "---%s:(.*)---" % constants.DEFAULT_NAME
|
|
||||||
DEFAULT_CSV_STREAM_FILE_FORMATTER = (
|
|
||||||
"---%s:" % constants.DEFAULT_NAME + "%s---%s"
|
|
||||||
)
|
|
||||||
DEFAULT_NEWLINE = "\r\n"
|
|
||||||
BOM_LITTLE_ENDIAN = b"\xff\xfe"
|
|
||||||
BOM_BIG_ENDIAN = b"\xfe\ff"
|
|
||||||
LITTLE_ENDIAN = 0
|
|
||||||
BIG_ENDIAN = 1
|
|
||||||
|
|
||||||
|
|
||||||
class CSVMemoryMapIterator(compact.Iterator):
|
|
||||||
"""
|
|
||||||
Wrapper class for mmap object
|
|
||||||
|
|
||||||
mmap object does not handle encoding at all. This class
|
|
||||||
provide the necessary transcoding for utf-8, utf-16 and utf-32
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, mmap_obj, encoding):
|
|
||||||
self.__mmap_obj = mmap_obj
|
|
||||||
self.__encoding = encoding
|
|
||||||
self.__count = 0
|
|
||||||
self.__endian = LITTLE_ENDIAN
|
|
||||||
if encoding == "utf-8":
|
|
||||||
# ..\r\x00\n
|
|
||||||
# \x00\x..
|
|
||||||
self.__zeros_left_in_2_row = 0
|
|
||||||
elif encoding == "utf-16":
|
|
||||||
# ..\r\x00\n
|
|
||||||
# \x00\x..
|
|
||||||
self.__zeros_left_in_2_row = 1
|
|
||||||
elif encoding == "utf-32":
|
|
||||||
# \r\x00\x00\x00\n
|
|
||||||
# \x00\x00\x00\x..
|
|
||||||
self.__zeros_left_in_2_row = 3
|
|
||||||
elif encoding == "utf-32-be" or encoding == "utf-16-be":
|
|
||||||
self.__zeros_left_in_2_row = 0
|
|
||||||
self.__endian = BIG_ENDIAN
|
|
||||||
elif encoding == "utf-32-le":
|
|
||||||
self.__zeros_left_in_2_row = 3
|
|
||||||
self.__endian = LITTLE_ENDIAN
|
|
||||||
elif encoding == "utf-16-le":
|
|
||||||
self.__zeros_left_in_2_row = 1
|
|
||||||
self.__endian = LITTLE_ENDIAN
|
|
||||||
else:
|
|
||||||
raise Exception("Encoding %s is not supported" % encoding)
|
|
||||||
|
|
||||||
def __iter__(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __next__(self):
|
|
||||||
line = self.__mmap_obj.readline()
|
|
||||||
if self.__count == 0:
|
|
||||||
utf_16_32 = (
|
|
||||||
self.__encoding == "utf-16" or self.__encoding == "utf-32"
|
|
||||||
)
|
|
||||||
if utf_16_32:
|
|
||||||
bom_header = line[:2]
|
|
||||||
if bom_header == BOM_BIG_ENDIAN:
|
|
||||||
self.__endian = BIG_ENDIAN
|
|
||||||
elif self.__endian == LITTLE_ENDIAN:
|
|
||||||
line = line[self.__zeros_left_in_2_row :] # noqa: E203
|
|
||||||
if self.__endian == LITTLE_ENDIAN:
|
|
||||||
line = line.rstrip()
|
|
||||||
line = line.decode(self.__encoding)
|
|
||||||
self.__count += 1
|
|
||||||
if line == "":
|
|
||||||
raise StopIteration
|
|
||||||
|
|
||||||
if compact.PY2:
|
|
||||||
# python 2 requires utf-8 encoded string for reading
|
|
||||||
line = line.encode("utf-8")
|
|
||||||
return line
|
|
||||||
|
|
||||||
|
|
||||||
class UTF8Recorder(compact.Iterator):
|
|
||||||
"""
|
|
||||||
Iterator that reads an encoded stream and reencodes the input to UTF-8.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, file_handle, encoding):
|
|
||||||
self.__file_handle = file_handle
|
|
||||||
self.reader = codecs.getreader(encoding)(file_handle)
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
self.__file_handle.close()
|
|
||||||
|
|
||||||
def __iter__(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __next__(self):
|
|
||||||
# python 2 requires utf-8 encoded string for reading
|
|
||||||
line = next(self.reader).encode("utf-8")
|
|
||||||
return line
|
|
||||||
|
|
||||||
|
|
||||||
class CSVSheetReader(SheetReader):
|
|
||||||
""" generic csv file reader"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
sheet,
|
|
||||||
encoding="utf-8",
|
|
||||||
auto_detect_float=True,
|
|
||||||
ignore_infinity=True,
|
|
||||||
auto_detect_int=True,
|
|
||||||
auto_detect_datetime=True,
|
|
||||||
pep_0515_off=True,
|
|
||||||
ignore_nan_text=False,
|
|
||||||
default_float_nan=None,
|
|
||||||
**keywords
|
|
||||||
):
|
|
||||||
SheetReader.__init__(self, sheet, **keywords)
|
|
||||||
self._encoding = encoding
|
|
||||||
self.__auto_detect_int = auto_detect_int
|
|
||||||
self.__auto_detect_float = auto_detect_float
|
|
||||||
self.__ignore_infinity = ignore_infinity
|
|
||||||
self.__auto_detect_datetime = auto_detect_datetime
|
|
||||||
self.__file_handle = None
|
|
||||||
self.__pep_0515_off = pep_0515_off
|
|
||||||
self.__ignore_nan_text = ignore_nan_text
|
|
||||||
self.__default_float_nan = default_float_nan
|
|
||||||
|
|
||||||
def get_file_handle(self):
|
|
||||||
""" return me unicde reader for csv """
|
|
||||||
raise NotImplementedError("Please implement get_file_handle()")
|
|
||||||
|
|
||||||
def row_iterator(self):
|
|
||||||
self.__file_handle = self.get_file_handle()
|
|
||||||
return csv.reader(self.__file_handle, **self._keywords)
|
|
||||||
|
|
||||||
def column_iterator(self, row):
|
|
||||||
for element in row:
|
|
||||||
if compact.PY2:
|
|
||||||
element = element.decode("utf-8")
|
|
||||||
if element is not None and element != "":
|
|
||||||
element = self.__convert_cell(element)
|
|
||||||
yield element
|
|
||||||
|
|
||||||
def __convert_cell(self, csv_cell_text):
|
|
||||||
ret = None
|
|
||||||
if self.__auto_detect_int:
|
|
||||||
ret = service.detect_int_value(csv_cell_text, self.__pep_0515_off)
|
|
||||||
if ret is None and self.__auto_detect_float:
|
|
||||||
ret = service.detect_float_value(
|
|
||||||
csv_cell_text,
|
|
||||||
self.__pep_0515_off,
|
|
||||||
ignore_nan_text=self.__ignore_nan_text,
|
|
||||||
default_float_nan=self.__default_float_nan,
|
|
||||||
)
|
|
||||||
shall_we_ignore_the_conversion = (
|
|
||||||
ret in [float("inf"), float("-inf")]
|
|
||||||
) and self.__ignore_infinity
|
|
||||||
if shall_we_ignore_the_conversion:
|
|
||||||
ret = None
|
|
||||||
if ret is None and self.__auto_detect_datetime:
|
|
||||||
ret = service.detect_date_value(csv_cell_text)
|
|
||||||
if ret is None:
|
|
||||||
ret = csv_cell_text
|
|
||||||
return ret
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
if self.__file_handle:
|
|
||||||
self.__file_handle.close()
|
|
||||||
|
|
||||||
|
|
||||||
# else: means the generator has been run
|
|
||||||
# yes, no run, no file open.
|
|
||||||
|
|
||||||
|
|
||||||
class CSVFileReader(CSVSheetReader):
|
|
||||||
""" read csv from phyical file """
|
|
||||||
|
|
||||||
def get_file_handle(self):
|
|
||||||
unicode_reader = None
|
|
||||||
if compact.PY2:
|
|
||||||
file_handle = open(self._native_sheet.payload, "rb")
|
|
||||||
unicode_reader = UTF8Recorder(file_handle, self._encoding)
|
|
||||||
else:
|
|
||||||
unicode_reader = open(
|
|
||||||
self._native_sheet.payload, "r", encoding=self._encoding
|
|
||||||
)
|
|
||||||
return unicode_reader
|
|
||||||
|
|
||||||
|
|
||||||
class CSVinMemoryReader(CSVSheetReader):
|
|
||||||
""" read csv file from memory """
|
|
||||||
|
|
||||||
def get_file_handle(self):
|
|
||||||
unicode_reader = None
|
|
||||||
if compact.PY2:
|
|
||||||
if hasattr(self._native_sheet.payload, "read"):
|
|
||||||
unicode_reader = UTF8Recorder(
|
|
||||||
self._native_sheet.payload, self._encoding
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
unicode_reader = self._native_sheet.payload
|
|
||||||
else:
|
|
||||||
if isinstance(self._native_sheet.payload, compact.BytesIO):
|
|
||||||
# please note that
|
|
||||||
# if the end developer feed us bytesio in python3
|
|
||||||
# we will do the conversion to StriongIO but that
|
|
||||||
# comes at a cost.
|
|
||||||
content = self._native_sheet.payload.read()
|
|
||||||
unicode_reader = compact.StringIO(
|
|
||||||
content.decode(self._encoding)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
unicode_reader = self._native_sheet.payload
|
|
||||||
|
|
||||||
return unicode_reader
|
|
||||||
|
|
||||||
|
|
||||||
class CSVBookReader(BookReader):
|
|
||||||
""" read csv file """
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
BookReader.__init__(self)
|
|
||||||
self._file_type = constants.FILE_FORMAT_CSV
|
|
||||||
self._file_content = None
|
|
||||||
self.__load_from_memory_flag = False
|
|
||||||
self.__line_terminator = constants.DEFAULT_CSV_NEWLINE
|
|
||||||
self.__sheet_name = None
|
|
||||||
self.__sheet_index = None
|
|
||||||
self.__multiple_sheets = False
|
|
||||||
self.__readers = []
|
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
|
||||||
BookReader.open(self, file_name, **keywords)
|
|
||||||
self._native_book = self._load_from_file()
|
|
||||||
|
|
||||||
def open_stream(self, file_stream, multiple_sheets=False, **keywords):
|
|
||||||
BookReader.open_stream(self, file_stream, **keywords)
|
|
||||||
self.__multiple_sheets = multiple_sheets
|
|
||||||
self._native_book = self._load_from_stream()
|
|
||||||
|
|
||||||
def open_content(self, file_content, **keywords):
|
|
||||||
try:
|
|
||||||
import mmap
|
|
||||||
|
|
||||||
encoding = keywords.get("encoding", "utf-8")
|
|
||||||
if isinstance(file_content, mmap.mmap):
|
|
||||||
# load from mmap
|
|
||||||
self.__multiple_sheets = keywords.get("multiple_sheets", False)
|
|
||||||
self._file_stream = CSVMemoryMapIterator(
|
|
||||||
file_content, encoding
|
|
||||||
)
|
|
||||||
self._keywords = keywords
|
|
||||||
self._native_book = self._load_from_stream()
|
|
||||||
else:
|
|
||||||
if compact.PY3_ABOVE:
|
|
||||||
if isinstance(file_content, bytes):
|
|
||||||
file_content = file_content.decode(encoding)
|
|
||||||
# else python 2.7 does not care about bytes nor str
|
|
||||||
BookReader.open_content(self, file_content, **keywords)
|
|
||||||
except ImportError:
|
|
||||||
# python 2.6 or Google app engine
|
|
||||||
BookReader.open_content(self, file_content, **keywords)
|
|
||||||
|
|
||||||
def read_sheet(self, native_sheet):
|
|
||||||
if self.__load_from_memory_flag:
|
|
||||||
reader = CSVinMemoryReader(native_sheet, **self._keywords)
|
|
||||||
else:
|
|
||||||
reader = CSVFileReader(native_sheet, **self._keywords)
|
|
||||||
self.__readers.append(reader)
|
|
||||||
return reader.to_array()
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
for reader in self.__readers:
|
|
||||||
reader.close()
|
|
||||||
|
|
||||||
def _load_from_stream(self):
|
|
||||||
"""Load content from memory
|
|
||||||
|
|
||||||
:params stream file_content: the actual file content in memory
|
|
||||||
:returns: a book
|
|
||||||
"""
|
|
||||||
self.__load_from_memory_flag = True
|
|
||||||
self.__line_terminator = self._keywords.get(
|
|
||||||
constants.KEYWORD_LINE_TERMINATOR, self.__line_terminator
|
|
||||||
)
|
|
||||||
separator = DEFAULT_SHEET_SEPARATOR_FORMATTER % self.__line_terminator
|
|
||||||
if self.__multiple_sheets:
|
|
||||||
# will be slow for large files
|
|
||||||
self._file_stream.seek(0)
|
|
||||||
content = self._file_stream.read()
|
|
||||||
sheets = content.split(separator)
|
|
||||||
named_contents = []
|
|
||||||
for sheet in sheets:
|
|
||||||
if sheet == "": # skip empty named sheet
|
|
||||||
continue
|
|
||||||
|
|
||||||
lines = sheet.split(self.__line_terminator)
|
|
||||||
result = re.match(constants.SEPARATOR_MATCHER, lines[0])
|
|
||||||
new_content = "\n".join(lines[1:])
|
|
||||||
new_sheet = NamedContent(
|
|
||||||
result.group(1), compact.StringIO(new_content)
|
|
||||||
)
|
|
||||||
named_contents.append(new_sheet)
|
|
||||||
return named_contents
|
|
||||||
|
|
||||||
else:
|
|
||||||
if hasattr(self._file_stream, "seek"):
|
|
||||||
self._file_stream.seek(0)
|
|
||||||
return [NamedContent(self._file_type, self._file_stream)]
|
|
||||||
|
|
||||||
def _load_from_file(self):
|
|
||||||
"""Load content from a file
|
|
||||||
|
|
||||||
:params str filename: an accessible file path
|
|
||||||
:returns: a book
|
|
||||||
"""
|
|
||||||
self.__line_terminator = self._keywords.get(
|
|
||||||
constants.KEYWORD_LINE_TERMINATOR, self.__line_terminator
|
|
||||||
)
|
|
||||||
names = os.path.splitext(self._file_name)
|
|
||||||
filepattern = "%s%s*%s*%s" % (
|
|
||||||
names[0],
|
|
||||||
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
|
||||||
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
|
||||||
names[1],
|
|
||||||
)
|
|
||||||
filelist = glob.glob(filepattern)
|
|
||||||
if len(filelist) == 0:
|
|
||||||
file_parts = os.path.split(self._file_name)
|
|
||||||
return [NamedContent(file_parts[-1], self._file_name)]
|
|
||||||
|
|
||||||
else:
|
|
||||||
matcher = "%s%s(.*)%s(.*)%s" % (
|
|
||||||
names[0],
|
|
||||||
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
|
||||||
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
|
||||||
names[1],
|
|
||||||
)
|
|
||||||
tmp_file_list = []
|
|
||||||
for filen in filelist:
|
|
||||||
result = re.match(matcher, filen)
|
|
||||||
tmp_file_list.append((result.group(1), result.group(2), filen))
|
|
||||||
ret = []
|
|
||||||
for lsheetname, index, filen in sorted(
|
|
||||||
tmp_file_list, key=lambda row: row[1]
|
|
||||||
):
|
|
||||||
ret.append(NamedContent(lsheetname, filen))
|
|
||||||
return ret
|
|
|
@ -4,69 +4,56 @@
|
||||||
|
|
||||||
The lower level csvz file format handler.
|
The lower level csvz file format handler.
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
import zipfile
|
import zipfile
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
from pyexcel_io._compact import StringIO, PY2
|
import chardet
|
||||||
from pyexcel_io.book import BookReader
|
from pyexcel_io import constants
|
||||||
from pyexcel_io.constants import FILE_FORMAT_CSVZ
|
from pyexcel_io.sheet import NamedContent
|
||||||
|
from pyexcel_io._compact import StringIO
|
||||||
from .csvr import CSVinMemoryReader, NamedContent
|
from pyexcel_io.readers.csv_sheet import CSVinMemoryReader
|
||||||
|
from pyexcel_io.plugin_api.abstract_reader import IReader
|
||||||
|
|
||||||
|
|
||||||
class CSVZipBookReader(BookReader):
|
class FileReader(IReader):
|
||||||
"""csvz reader
|
def __init__(self, file_alike_object, file_type, **keywords):
|
||||||
|
self.content_array = []
|
||||||
Read zipped csv file that was zipped up by pyexcel-io. It support
|
|
||||||
single csv file and multiple csv files.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
BookReader.__init__(self)
|
|
||||||
self._file_type = FILE_FORMAT_CSVZ
|
|
||||||
self.zipfile = None
|
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
|
||||||
BookReader.open(self, file_name, **keywords)
|
|
||||||
self._native_book = self._load_from_file_alike_object(self._file_name)
|
|
||||||
|
|
||||||
def open_stream(self, file_stream, **keywords):
|
|
||||||
BookReader.open_stream(self, file_stream, **keywords)
|
|
||||||
self._native_book = self._load_from_file_alike_object(
|
|
||||||
self._file_stream
|
|
||||||
)
|
|
||||||
|
|
||||||
def read_sheet(self, native_sheet):
|
|
||||||
content = self.zipfile.read(native_sheet.payload)
|
|
||||||
if PY2:
|
|
||||||
sheet = StringIO(content)
|
|
||||||
else:
|
|
||||||
sheet = StringIO(content.decode("utf-8"))
|
|
||||||
|
|
||||||
reader = CSVinMemoryReader(
|
|
||||||
NamedContent(native_sheet.name, sheet), **self._keywords
|
|
||||||
)
|
|
||||||
return reader.to_array()
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
if self.zipfile:
|
|
||||||
self.zipfile.close()
|
|
||||||
|
|
||||||
def _load_from_file_alike_object(self, file_alike_object):
|
|
||||||
try:
|
try:
|
||||||
self.zipfile = zipfile.ZipFile(file_alike_object, "r")
|
self.zipfile = zipfile.ZipFile(file_alike_object, "r")
|
||||||
sheets = [
|
sheets = [
|
||||||
NamedContent(_get_sheet_name(name), name)
|
NamedContent(_get_sheet_name(name), name)
|
||||||
for name in self.zipfile.namelist()
|
for name in self.zipfile.namelist()
|
||||||
]
|
]
|
||||||
return sheets
|
self.content_array = sheets
|
||||||
|
self.keywords = keywords
|
||||||
|
if file_type == constants.FILE_FORMAT_TSVZ:
|
||||||
|
self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
||||||
|
|
||||||
except zipfile.BadZipfile:
|
except zipfile.BadZipfile:
|
||||||
print("StringIO instance was passed by any chance?")
|
print("StringIO instance was passed by any chance?")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
if self.zipfile:
|
||||||
|
self.zipfile.close()
|
||||||
|
|
||||||
|
def read_sheet(self, index):
|
||||||
|
name = self.content_array[index].name
|
||||||
|
content = self.zipfile.read(self.content_array[index].payload)
|
||||||
|
encoding_guess = chardet.detect(content)
|
||||||
|
sheet = StringIO(content.decode(encoding_guess["encoding"]))
|
||||||
|
|
||||||
|
return CSVinMemoryReader(NamedContent(name, sheet), **self.keywords)
|
||||||
|
|
||||||
|
|
||||||
|
class ContentReader(FileReader):
|
||||||
|
def __init__(self, file_content, file_type, **keywords):
|
||||||
|
io = BytesIO(file_content)
|
||||||
|
super().__init__(io, file_type, **keywords)
|
||||||
|
|
||||||
|
|
||||||
def _get_sheet_name(filename):
|
def _get_sheet_name(filename):
|
||||||
len_of_a_dot = 1
|
len_of_a_dot = 1
|
||||||
|
|
|
@ -1,27 +0,0 @@
|
||||||
"""
|
|
||||||
pyexcel_io.readers.tsv
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
The lower level tsv file format handler.
|
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
|
||||||
:license: New BSD License, see LICENSE for more details
|
|
||||||
"""
|
|
||||||
import pyexcel_io.constants as constants
|
|
||||||
from .csvr import CSVBookReader
|
|
||||||
|
|
||||||
|
|
||||||
class TSVBookReader(CSVBookReader):
|
|
||||||
""" Read tab separated values """
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
CSVBookReader.__init__(self)
|
|
||||||
self._file_type = constants.FILE_FORMAT_TSV
|
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
|
||||||
keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
|
||||||
CSVBookReader.open(self, file_name, **keywords)
|
|
||||||
|
|
||||||
def open_stream(self, file_content, **keywords):
|
|
||||||
keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
|
||||||
CSVBookReader.open_stream(self, file_content, **keywords)
|
|
|
@ -1,31 +0,0 @@
|
||||||
"""
|
|
||||||
pyexcel_io.fileformat.tsvz
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
The lower level tsvz file format handler.
|
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
|
||||||
:license: New BSD License, see LICENSE for more details
|
|
||||||
"""
|
|
||||||
from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT
|
|
||||||
|
|
||||||
from .csvz import CSVZipBookReader
|
|
||||||
|
|
||||||
|
|
||||||
class TSVZipBookReader(CSVZipBookReader):
|
|
||||||
""" read zipped tab separated value file
|
|
||||||
|
|
||||||
it supports single tsv file and mulitple tsv files
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
CSVZipBookReader.__init__(self)
|
|
||||||
self._file_type = FILE_FORMAT_TSVZ
|
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
|
||||||
keywords["dialect"] = KEYWORD_TSV_DIALECT
|
|
||||||
CSVZipBookReader.open(self, file_name, **keywords)
|
|
||||||
|
|
||||||
def open_stream(self, file_content, **keywords):
|
|
||||||
keywords["dialect"] = KEYWORD_TSV_DIALECT
|
|
||||||
CSVZipBookReader.open_stream(self, file_content, **keywords)
|
|
|
@ -4,16 +4,14 @@
|
||||||
|
|
||||||
provide service code to downstream projects
|
provide service code to downstream projects
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
import re
|
import re
|
||||||
import math
|
import math
|
||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
from pyexcel_io._compact import PY2
|
from pyexcel_io import constants, exceptions
|
||||||
from pyexcel_io import constants
|
|
||||||
from pyexcel_io import exceptions
|
|
||||||
|
|
||||||
|
|
||||||
def has_no_digits_in_float(value):
|
def has_no_digits_in_float(value):
|
||||||
|
@ -175,13 +173,10 @@ ODS_WRITE_FORMAT_COVERSION = {
|
||||||
datetime.date: "date",
|
datetime.date: "date",
|
||||||
datetime.time: "time",
|
datetime.time: "time",
|
||||||
datetime.timedelta: "timedelta",
|
datetime.timedelta: "timedelta",
|
||||||
|
datetime.datetime: "datetime",
|
||||||
bool: "boolean",
|
bool: "boolean",
|
||||||
}
|
}
|
||||||
|
|
||||||
if PY2:
|
|
||||||
ODS_WRITE_FORMAT_COVERSION[unicode] = "string" # noqa: F821
|
|
||||||
ODS_WRITE_FORMAT_COVERSION[long] = "throw_exception" # noqa: F821
|
|
||||||
|
|
||||||
|
|
||||||
VALUE_CONVERTERS = {
|
VALUE_CONVERTERS = {
|
||||||
"float": float_value,
|
"float": float_value,
|
||||||
|
@ -234,7 +229,7 @@ ODS_VALUE_CONVERTERS = {
|
||||||
"boolean": ods_bool_value,
|
"boolean": ods_bool_value,
|
||||||
"timedelta": ods_timedelta_value,
|
"timedelta": ods_timedelta_value,
|
||||||
"float": ods_float_value,
|
"float": ods_float_value,
|
||||||
"throw_exception": throw_exception
|
"long": ods_float_value,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -4,22 +4,13 @@
|
||||||
|
|
||||||
The io interface to file extensions
|
The io interface to file extensions
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io._compact import irange
|
|
||||||
from pyexcel_io.utils import _index_filter
|
|
||||||
import pyexcel_io.constants as constants
|
import pyexcel_io.constants as constants
|
||||||
|
from pyexcel_io.utils import _index_filter
|
||||||
|
from pyexcel_io._compact import irange
|
||||||
class NamedContent(object):
|
from pyexcel_io.plugin_api import NamedContent # noqa: F401
|
||||||
"""
|
|
||||||
Helper class for content that does not have a name
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, name, payload):
|
|
||||||
self.name = name
|
|
||||||
self.payload = payload
|
|
||||||
|
|
||||||
|
|
||||||
class SheetReader(object):
|
class SheetReader(object):
|
||||||
|
@ -38,11 +29,12 @@ class SheetReader(object):
|
||||||
skip_column_func=None,
|
skip_column_func=None,
|
||||||
skip_empty_rows=False,
|
skip_empty_rows=False,
|
||||||
row_renderer=None,
|
row_renderer=None,
|
||||||
**keywords
|
keep_trailing_empty_cells=False,
|
||||||
|
**deprecated_use_of_keywords_here
|
||||||
):
|
):
|
||||||
self._native_sheet = sheet
|
self._native_sheet = sheet
|
||||||
self._keywords = {}
|
self._keywords = {}
|
||||||
self._keywords.update(keywords)
|
self._keywords.update(deprecated_use_of_keywords_here)
|
||||||
self._start_row = start_row
|
self._start_row = start_row
|
||||||
self._row_limit = row_limit
|
self._row_limit = row_limit
|
||||||
self._start_column = start_column
|
self._start_column = start_column
|
||||||
|
@ -51,6 +43,7 @@ class SheetReader(object):
|
||||||
self._skip_column = _index_filter
|
self._skip_column = _index_filter
|
||||||
self._skip_empty_rows = skip_empty_rows
|
self._skip_empty_rows = skip_empty_rows
|
||||||
self._row_renderer = row_renderer
|
self._row_renderer = row_renderer
|
||||||
|
self.keep_trailing_empty_cells = keep_trailing_empty_cells
|
||||||
|
|
||||||
if skip_row_func:
|
if skip_row_func:
|
||||||
self._skip_row = skip_row_func
|
self._skip_row = skip_row_func
|
||||||
|
@ -58,8 +51,7 @@ class SheetReader(object):
|
||||||
self._skip_column = skip_column_func
|
self._skip_column = skip_column_func
|
||||||
|
|
||||||
def to_array(self):
|
def to_array(self):
|
||||||
"""2 dimentional representation of the content
|
"""2 dimentional representation of the content"""
|
||||||
"""
|
|
||||||
for row_index, row in enumerate(self.row_iterator()):
|
for row_index, row in enumerate(self.row_iterator()):
|
||||||
row_position = self._skip_row(
|
row_position = self._skip_row(
|
||||||
row_index, self._start_row, self._row_limit
|
row_index, self._start_row, self._row_limit
|
||||||
|
@ -85,10 +77,13 @@ class SheetReader(object):
|
||||||
elif column_position == constants.STOP_ITERATION:
|
elif column_position == constants.STOP_ITERATION:
|
||||||
break
|
break
|
||||||
|
|
||||||
tmp_row.append(cell_value)
|
if self.keep_trailing_empty_cells:
|
||||||
if cell_value is not None and cell_value != "":
|
return_row.append(cell_value)
|
||||||
return_row += tmp_row
|
else:
|
||||||
tmp_row = []
|
tmp_row.append(cell_value)
|
||||||
|
if cell_value is not None and cell_value != "":
|
||||||
|
return_row += tmp_row
|
||||||
|
tmp_row = []
|
||||||
if self._skip_empty_rows and len(return_row) < 1:
|
if self._skip_empty_rows and len(return_row) < 1:
|
||||||
# we by-pass next yeild here
|
# we by-pass next yeild here
|
||||||
# because it is an empty row
|
# because it is an empty row
|
||||||
|
|
|
@ -4,36 +4,45 @@
|
||||||
|
|
||||||
utility functions
|
utility functions
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
import pyexcel_io.constants as constants
|
import pyexcel_io.constants as constants
|
||||||
|
|
||||||
|
|
||||||
XLS_PLUGIN = "pyexcel-xls"
|
XLS_PLUGIN = "pyexcel-xls"
|
||||||
XLSX_PLUGIN = "pyexcel-xlsx"
|
XLSX_PLUGIN = "pyexcel-xlsx"
|
||||||
ODS_PLUGIN = "pyexcel-ods"
|
ODS_PLUGIN = "pyexcel-ods"
|
||||||
ODS3_PLUGIN = "pyexcel-ods3"
|
ODS3_PLUGIN = "pyexcel-ods3"
|
||||||
|
ODSR_PLUGIN = "pyexcel-odsr"
|
||||||
|
ODSW_PLUGIN = "pyexcel-odsw"
|
||||||
|
XLSXR_PLUGIN = "pyexcel-xlsxr"
|
||||||
XLSXW_PLUGIN = "pyexcel-xlsxw"
|
XLSXW_PLUGIN = "pyexcel-xlsxw"
|
||||||
|
XLSBR_PLUGIN = "pyexcel-xlsbr"
|
||||||
|
HTMLR_PLUGIN = "pyexcel-htmlr"
|
||||||
|
PDFR_PLUGIN = "pyexcel-pdfr"
|
||||||
IO_ITSELF = "pyexcel-io"
|
IO_ITSELF = "pyexcel-io"
|
||||||
|
|
||||||
|
AVAILABLE_NEW_READERS = {}
|
||||||
|
|
||||||
AVAILABLE_READERS = {
|
AVAILABLE_READERS = {
|
||||||
|
constants.FILE_FORMAT_CSV: [IO_ITSELF],
|
||||||
constants.FILE_FORMAT_XLS: [XLS_PLUGIN],
|
constants.FILE_FORMAT_XLS: [XLS_PLUGIN],
|
||||||
constants.FILE_FORMAT_XLSX: [XLS_PLUGIN, XLSX_PLUGIN],
|
constants.FILE_FORMAT_XLSX: [XLS_PLUGIN, XLSX_PLUGIN],
|
||||||
constants.FILE_FORMAT_XLSM: [XLS_PLUGIN, XLSX_PLUGIN],
|
constants.FILE_FORMAT_XLSM: [XLS_PLUGIN, XLSX_PLUGIN],
|
||||||
constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN],
|
constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN, ODSR_PLUGIN],
|
||||||
constants.FILE_FORMAT_CSV: [IO_ITSELF],
|
|
||||||
constants.FILE_FORMAT_TSV: [IO_ITSELF],
|
constants.FILE_FORMAT_TSV: [IO_ITSELF],
|
||||||
constants.FILE_FORMAT_CSVZ: [IO_ITSELF],
|
constants.FILE_FORMAT_CSVZ: [IO_ITSELF],
|
||||||
constants.FILE_FORMAT_TSVZ: [IO_ITSELF],
|
constants.FILE_FORMAT_TSVZ: [IO_ITSELF],
|
||||||
|
constants.FILE_FORMAT_XLSB: [XLSBR_PLUGIN],
|
||||||
|
constants.FILE_FORMAT_HTML: [HTMLR_PLUGIN],
|
||||||
|
constants.FILE_FORMAT_PDF: [PDFR_PLUGIN],
|
||||||
}
|
}
|
||||||
|
|
||||||
AVAILABLE_WRITERS = {
|
AVAILABLE_WRITERS = {
|
||||||
constants.FILE_FORMAT_XLS: [XLS_PLUGIN],
|
constants.FILE_FORMAT_XLS: [XLS_PLUGIN],
|
||||||
constants.FILE_FORMAT_XLSX: [XLSX_PLUGIN, XLSXW_PLUGIN],
|
constants.FILE_FORMAT_XLSX: [XLSX_PLUGIN, XLSXW_PLUGIN],
|
||||||
constants.FILE_FORMAT_XLSM: [XLSX_PLUGIN],
|
constants.FILE_FORMAT_XLSM: [XLSX_PLUGIN],
|
||||||
constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN],
|
constants.FILE_FORMAT_ODS: [ODS_PLUGIN, ODS3_PLUGIN, ODSW_PLUGIN],
|
||||||
constants.FILE_FORMAT_CSV: [IO_ITSELF],
|
constants.FILE_FORMAT_CSV: [IO_ITSELF],
|
||||||
constants.FILE_FORMAT_TSV: [IO_ITSELF],
|
constants.FILE_FORMAT_TSV: [IO_ITSELF],
|
||||||
constants.FILE_FORMAT_CSVZ: [IO_ITSELF],
|
constants.FILE_FORMAT_CSVZ: [IO_ITSELF],
|
||||||
|
@ -60,10 +69,10 @@ def is_empty_array(array):
|
||||||
|
|
||||||
|
|
||||||
def swap_empty_string_for_none(array):
|
def swap_empty_string_for_none(array):
|
||||||
""" replace empty string fields with None """
|
"""replace empty string fields with None"""
|
||||||
|
|
||||||
def swap(value):
|
def swap(value):
|
||||||
""" change empty string to None """
|
"""change empty string to None"""
|
||||||
if value == "":
|
if value == "":
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,51 @@
|
||||||
|
from pyexcel_io.plugins import NEW_WRITERS
|
||||||
|
|
||||||
|
|
||||||
|
class Writer(object):
|
||||||
|
def __init__(self, file_type, library=None):
|
||||||
|
self.file_type = file_type
|
||||||
|
self.library = library
|
||||||
|
self.keyboards = None
|
||||||
|
# if you know which reader class to use, this attribute allows
|
||||||
|
# you to set reader class externally. Since there is no
|
||||||
|
# so call private field in Python, I am not going to create
|
||||||
|
# useless setter and getter functions like Java.
|
||||||
|
# in pyexcel, this attribute is mainly used for testing
|
||||||
|
self.writer_class = None
|
||||||
|
|
||||||
|
def open(self, file_name, **keywords):
|
||||||
|
if self.writer_class is None:
|
||||||
|
self.writer_class = NEW_WRITERS.get_a_plugin(
|
||||||
|
self.file_type, library=self.library, location="file"
|
||||||
|
)
|
||||||
|
self.writer = self.writer_class(file_name, self.file_type, **keywords)
|
||||||
|
|
||||||
|
def open_content(self, file_stream, **keywords):
|
||||||
|
if self.writer_class is None:
|
||||||
|
self.writer_class = NEW_WRITERS.get_a_plugin(
|
||||||
|
self.file_type, library=self.library, location="content"
|
||||||
|
)
|
||||||
|
self.writer = self.writer_class(
|
||||||
|
file_stream, self.file_type, **keywords
|
||||||
|
)
|
||||||
|
|
||||||
|
def open_stream(self, file_stream, **keywords):
|
||||||
|
if self.writer_class is None:
|
||||||
|
self.writer_class = NEW_WRITERS.get_a_plugin(
|
||||||
|
self.file_type, library=self.library, location="memory"
|
||||||
|
)
|
||||||
|
self.writer = self.writer_class(
|
||||||
|
file_stream, self.file_type, **keywords
|
||||||
|
)
|
||||||
|
|
||||||
|
def write(self, incoming_dict):
|
||||||
|
self.writer.write(incoming_dict)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.writer.close()
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, a_type, value, traceback):
|
||||||
|
self.close()
|
|
@ -4,26 +4,24 @@
|
||||||
|
|
||||||
file writers
|
file writers
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
:license: New BSD License, see LICENSE for more details
|
:license: New BSD License, see LICENSE for more details
|
||||||
"""
|
"""
|
||||||
from pyexcel_io.plugins import IOPluginInfoChain
|
from pyexcel_io.plugins import IOPluginInfoChainV2
|
||||||
|
|
||||||
|
IOPluginInfoChainV2(__name__).add_a_writer(
|
||||||
IOPluginInfoChain(__name__).add_a_writer(
|
relative_plugin_class_path="csv_in_file.CsvFileWriter",
|
||||||
relative_plugin_class_path="csvw.CSVBookWriter",
|
locations=["file", "content"],
|
||||||
file_types=["csv"],
|
file_types=["csv", "tsv"],
|
||||||
stream_type="text",
|
stream_type="text",
|
||||||
).add_a_writer(
|
).add_a_writer(
|
||||||
relative_plugin_class_path="tsv.TSVBookWriter",
|
relative_plugin_class_path="csv_in_memory.CsvMemoryWriter",
|
||||||
file_types=["tsv"],
|
locations=["memory"],
|
||||||
|
file_types=["csv", "tsv"],
|
||||||
stream_type="text",
|
stream_type="text",
|
||||||
).add_a_writer(
|
).add_a_writer(
|
||||||
relative_plugin_class_path="csvz.CSVZipBookWriter",
|
relative_plugin_class_path="csvz_writer.CsvZipWriter",
|
||||||
file_types=["csvz"],
|
locations=["memory", "file", "content"],
|
||||||
stream_type="binary",
|
file_types=["csvz", "tsvz"],
|
||||||
).add_a_writer(
|
|
||||||
relative_plugin_class_path="tsvz.TSVZipBookWriter",
|
|
||||||
file_types=["tsvz"],
|
|
||||||
stream_type="binary",
|
stream_type="binary",
|
||||||
)
|
)
|
||||||
|
|
|
@ -0,0 +1,29 @@
|
||||||
|
from pyexcel_io import constants
|
||||||
|
from pyexcel_io.plugin_api import IWriter
|
||||||
|
from pyexcel_io.writers.csv_sheet import CSVFileWriter
|
||||||
|
|
||||||
|
|
||||||
|
class CsvFileWriter(IWriter):
|
||||||
|
def __init__(self, file_alike_object, file_type, **keywords):
|
||||||
|
self._file_alike_object = file_alike_object
|
||||||
|
self._keywords = keywords
|
||||||
|
if file_type == constants.FILE_FORMAT_TSV:
|
||||||
|
self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
||||||
|
self.__index = 0
|
||||||
|
self.handlers = []
|
||||||
|
|
||||||
|
def create_sheet(self, name):
|
||||||
|
writer = CSVFileWriter(
|
||||||
|
self._file_alike_object,
|
||||||
|
name,
|
||||||
|
sheet_index=self.__index,
|
||||||
|
**self._keywords
|
||||||
|
)
|
||||||
|
self.__index = self.__index + 1
|
||||||
|
self.handlers.append(writer)
|
||||||
|
return writer
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
for writer in self.handlers:
|
||||||
|
writer.close()
|
||||||
|
self.handlers = []
|
|
@ -0,0 +1,26 @@
|
||||||
|
from pyexcel_io import constants
|
||||||
|
from pyexcel_io.plugin_api import IWriter
|
||||||
|
from pyexcel_io.writers.csv_sheet import CSVMemoryWriter
|
||||||
|
|
||||||
|
|
||||||
|
class CsvMemoryWriter(IWriter):
|
||||||
|
def __init__(self, file_alike_object, file_type, **keywords):
|
||||||
|
self._file_alike_object = file_alike_object
|
||||||
|
self._keywords = keywords
|
||||||
|
if file_type == constants.FILE_FORMAT_TSV:
|
||||||
|
self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
||||||
|
self.__index = 0
|
||||||
|
|
||||||
|
def create_sheet(self, name):
|
||||||
|
writer_class = CSVMemoryWriter
|
||||||
|
writer = writer_class(
|
||||||
|
self._file_alike_object,
|
||||||
|
name,
|
||||||
|
sheet_index=self.__index,
|
||||||
|
**self._keywords
|
||||||
|
)
|
||||||
|
self.__index = self.__index + 1
|
||||||
|
return writer
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
|
@ -0,0 +1,98 @@
|
||||||
|
"""
|
||||||
|
pyexcel_io.writers.csv_sheet
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
The lower level csv file format writer
|
||||||
|
|
||||||
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
|
:license: New BSD License, see LICENSE for more details
|
||||||
|
"""
|
||||||
|
import csv
|
||||||
|
|
||||||
|
import pyexcel_io.constants as constants
|
||||||
|
from pyexcel_io.plugin_api import ISheetWriter
|
||||||
|
|
||||||
|
|
||||||
|
class CSVFileWriter(ISheetWriter):
|
||||||
|
"""
|
||||||
|
csv file writer
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
filename,
|
||||||
|
name,
|
||||||
|
encoding="utf-8",
|
||||||
|
single_sheet_in_book=False,
|
||||||
|
sheet_index=None,
|
||||||
|
**keywords
|
||||||
|
):
|
||||||
|
self._encoding = encoding
|
||||||
|
self._sheet_name = name
|
||||||
|
if self._sheet_name is None or single_sheet_in_book:
|
||||||
|
self._sheet_name = constants.DEFAULT_SHEET_NAME
|
||||||
|
self._single_sheet_in_book = single_sheet_in_book
|
||||||
|
self.__line_terminator = constants.DEFAULT_CSV_NEWLINE
|
||||||
|
self._keywords = keywords
|
||||||
|
if constants.KEYWORD_LINE_TERMINATOR in keywords:
|
||||||
|
self.__line_terminator = keywords.get(
|
||||||
|
constants.KEYWORD_LINE_TERMINATOR
|
||||||
|
)
|
||||||
|
self._sheet_index = sheet_index
|
||||||
|
self.file_handle = None
|
||||||
|
self._native_book = filename
|
||||||
|
|
||||||
|
self.writer = self.get_writer()
|
||||||
|
|
||||||
|
def get_writer(self):
|
||||||
|
if self._sheet_name != constants.DEFAULT_SHEET_NAME:
|
||||||
|
names = self._native_book.split(".")
|
||||||
|
file_name = "%s%s%s%s%s.%s" % (
|
||||||
|
names[0],
|
||||||
|
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
||||||
|
self._sheet_name, # sheet name
|
||||||
|
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
||||||
|
self._sheet_index, # sheet index
|
||||||
|
names[1],
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
file_name = self._native_book
|
||||||
|
|
||||||
|
self.file_handle = open(
|
||||||
|
file_name, "w", newline="", encoding=self._encoding
|
||||||
|
)
|
||||||
|
return csv.writer(self.file_handle, **self._keywords)
|
||||||
|
|
||||||
|
def write_row(self, array):
|
||||||
|
"""
|
||||||
|
write a row into the file
|
||||||
|
"""
|
||||||
|
self.writer.writerow(array)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.file_handle.close()
|
||||||
|
|
||||||
|
|
||||||
|
class CSVMemoryWriter(CSVFileWriter):
|
||||||
|
"""Write csv to a memory stream"""
|
||||||
|
|
||||||
|
def get_writer(self):
|
||||||
|
self.file_handle = self._native_book
|
||||||
|
writer = csv.writer(self.file_handle, **self._keywords)
|
||||||
|
if not self._single_sheet_in_book:
|
||||||
|
writer.writerow(
|
||||||
|
[
|
||||||
|
constants.DEFAULT_CSV_STREAM_FILE_FORMATTER
|
||||||
|
% (self._sheet_name, "")
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return writer
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
if self._single_sheet_in_book:
|
||||||
|
# on purpose, the this is not done
|
||||||
|
# because the io stream can be used later
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
self.writer.writerow([constants.SEPARATOR_FORMATTER % ""])
|
|
@ -1,192 +0,0 @@
|
||||||
"""
|
|
||||||
pyexcel_io.writers.csvw
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
The lower level csv file format writer
|
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
|
||||||
:license: New BSD License, see LICENSE for more details
|
|
||||||
"""
|
|
||||||
import csv
|
|
||||||
import codecs
|
|
||||||
|
|
||||||
from pyexcel_io.book import BookWriter
|
|
||||||
from pyexcel_io.sheet import SheetWriter
|
|
||||||
import pyexcel_io._compact as compact
|
|
||||||
import pyexcel_io.constants as constants
|
|
||||||
|
|
||||||
|
|
||||||
class UnicodeWriter(object):
|
|
||||||
"""
|
|
||||||
A CSV writer which will write rows to CSV file "f",
|
|
||||||
which is encoded in the given encoding.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, file_handle, encoding="utf-8", **kwds):
|
|
||||||
# Redirect output to a queue
|
|
||||||
self.queue = compact.StringIO()
|
|
||||||
self.writer = csv.writer(self.queue, **kwds)
|
|
||||||
self.stream = file_handle
|
|
||||||
self.encoder = codecs.getincrementalencoder(encoding)()
|
|
||||||
|
|
||||||
def writerow(self, row):
|
|
||||||
""" write row into the csv file """
|
|
||||||
self.writer.writerow(
|
|
||||||
[compact.text_type(s).encode("utf-8") for s in row]
|
|
||||||
)
|
|
||||||
# Fetch UTF-8 output from the queue ...
|
|
||||||
data = self.queue.getvalue()
|
|
||||||
data = data.decode("utf-8")
|
|
||||||
# ... and reencode it into the target encoding
|
|
||||||
data = self.encoder.encode(data)
|
|
||||||
# write to the target stream
|
|
||||||
self.stream.write(data)
|
|
||||||
# empty queue
|
|
||||||
self.queue.truncate(0)
|
|
||||||
|
|
||||||
def writerows(self, rows):
|
|
||||||
""" write multiple rows into csv file """
|
|
||||||
for row in rows:
|
|
||||||
self.writerow(row)
|
|
||||||
|
|
||||||
|
|
||||||
class CSVSheetWriter(SheetWriter):
|
|
||||||
"""
|
|
||||||
csv file writer
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
filename,
|
|
||||||
name,
|
|
||||||
encoding="utf-8",
|
|
||||||
single_sheet_in_book=False,
|
|
||||||
sheet_index=None,
|
|
||||||
**keywords
|
|
||||||
):
|
|
||||||
self._encoding = encoding
|
|
||||||
self._sheet_name = name
|
|
||||||
self._single_sheet_in_book = single_sheet_in_book
|
|
||||||
self.__line_terminator = constants.DEFAULT_CSV_NEWLINE
|
|
||||||
if constants.KEYWORD_LINE_TERMINATOR in keywords:
|
|
||||||
self.__line_terminator = keywords.get(
|
|
||||||
constants.KEYWORD_LINE_TERMINATOR
|
|
||||||
)
|
|
||||||
if single_sheet_in_book:
|
|
||||||
self._sheet_name = None
|
|
||||||
self._sheet_index = sheet_index
|
|
||||||
self.writer = None
|
|
||||||
self.file_handle = None
|
|
||||||
SheetWriter.__init__(
|
|
||||||
self, filename, self._sheet_name, self._sheet_name, **keywords
|
|
||||||
)
|
|
||||||
|
|
||||||
def write_row(self, array):
|
|
||||||
"""
|
|
||||||
write a row into the file
|
|
||||||
"""
|
|
||||||
self.writer.writerow(array)
|
|
||||||
|
|
||||||
|
|
||||||
class CSVFileWriter(CSVSheetWriter):
|
|
||||||
""" Write csv to a physical file """
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
self.file_handle.close()
|
|
||||||
|
|
||||||
def set_sheet_name(self, name):
|
|
||||||
if name != constants.DEFAULT_SHEET_NAME:
|
|
||||||
names = self._native_book.split(".")
|
|
||||||
file_name = "%s%s%s%s%s.%s" % (
|
|
||||||
names[0],
|
|
||||||
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
|
||||||
name, # sheet name
|
|
||||||
constants.DEFAULT_MULTI_CSV_SEPARATOR,
|
|
||||||
self._sheet_index, # sheet index
|
|
||||||
names[1],
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
file_name = self._native_book
|
|
||||||
if compact.PY2:
|
|
||||||
self.file_handle = open(file_name, "wb")
|
|
||||||
self.writer = UnicodeWriter(
|
|
||||||
self.file_handle, encoding=self._encoding, **self._keywords
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self.file_handle = open(
|
|
||||||
file_name, "w", newline="", encoding=self._encoding
|
|
||||||
)
|
|
||||||
self.writer = csv.writer(self.file_handle, **self._keywords)
|
|
||||||
|
|
||||||
|
|
||||||
class CSVMemoryWriter(CSVSheetWriter):
|
|
||||||
""" Write csv to a memory stream """
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
filename,
|
|
||||||
name,
|
|
||||||
encoding="utf-8",
|
|
||||||
single_sheet_in_book=False,
|
|
||||||
sheet_index=None,
|
|
||||||
**keywords
|
|
||||||
):
|
|
||||||
CSVSheetWriter.__init__(
|
|
||||||
self,
|
|
||||||
filename,
|
|
||||||
name,
|
|
||||||
encoding=encoding,
|
|
||||||
single_sheet_in_book=single_sheet_in_book,
|
|
||||||
sheet_index=sheet_index,
|
|
||||||
**keywords
|
|
||||||
)
|
|
||||||
|
|
||||||
def set_sheet_name(self, name):
|
|
||||||
if compact.PY2:
|
|
||||||
self.file_handle = self._native_book
|
|
||||||
self.writer = UnicodeWriter(
|
|
||||||
self.file_handle, encoding=self._encoding, **self._keywords
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self.file_handle = self._native_book
|
|
||||||
self.writer = csv.writer(self.file_handle, **self._keywords)
|
|
||||||
if not self._single_sheet_in_book:
|
|
||||||
self.writer.writerow(
|
|
||||||
[
|
|
||||||
constants.DEFAULT_CSV_STREAM_FILE_FORMATTER
|
|
||||||
% (self._sheet_name, "")
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
if self._single_sheet_in_book:
|
|
||||||
# on purpose, the this is not done
|
|
||||||
# because the io stream can be used later
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
self.writer.writerow([constants.SEPARATOR_FORMATTER % ""])
|
|
||||||
|
|
||||||
|
|
||||||
class CSVBookWriter(BookWriter):
|
|
||||||
""" write csv with unicode support """
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
BookWriter.__init__(self)
|
|
||||||
self._file_type = constants.FILE_FORMAT_CSV
|
|
||||||
self.__index = 0
|
|
||||||
|
|
||||||
def create_sheet(self, name):
|
|
||||||
writer_class = None
|
|
||||||
if compact.is_string(type(self._file_alike_object)):
|
|
||||||
writer_class = CSVFileWriter
|
|
||||||
else:
|
|
||||||
writer_class = CSVMemoryWriter
|
|
||||||
writer = writer_class(
|
|
||||||
self._file_alike_object,
|
|
||||||
name,
|
|
||||||
sheet_index=self.__index,
|
|
||||||
**self._keywords
|
|
||||||
)
|
|
||||||
self.__index = self.__index + 1
|
|
||||||
return writer
|
|
|
@ -1,73 +0,0 @@
|
||||||
"""
|
|
||||||
pyexcel_io.fileformat.csvz
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
The lower level csvz file format handler.
|
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
|
||||||
:license: New BSD License, see LICENSE for more details
|
|
||||||
"""
|
|
||||||
import zipfile
|
|
||||||
|
|
||||||
from pyexcel_io._compact import StringIO, PY2
|
|
||||||
from pyexcel_io.book import BookWriter
|
|
||||||
from pyexcel_io.constants import DEFAULT_SHEET_NAME, FILE_FORMAT_CSVZ
|
|
||||||
|
|
||||||
from .csvw import CSVSheetWriter, UnicodeWriter
|
|
||||||
|
|
||||||
|
|
||||||
class CSVZipSheetWriter(CSVSheetWriter):
|
|
||||||
""" handle the zipfile interface """
|
|
||||||
|
|
||||||
def __init__(self, zipfile, sheetname, file_extension, **keywords):
|
|
||||||
self.file_extension = file_extension
|
|
||||||
keywords["single_sheet_in_book"] = False
|
|
||||||
CSVSheetWriter.__init__(self, zipfile, sheetname, **keywords)
|
|
||||||
|
|
||||||
def set_sheet_name(self, name):
|
|
||||||
self.content = StringIO()
|
|
||||||
if PY2:
|
|
||||||
self.writer = UnicodeWriter(
|
|
||||||
self.content, encoding=self._encoding, **self._keywords
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
import csv
|
|
||||||
|
|
||||||
self.writer = csv.writer(self.content, **self._keywords)
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
file_name = "%s.%s" % (self._native_sheet, self.file_extension)
|
|
||||||
self.content.seek(0)
|
|
||||||
self._native_book.writestr(file_name, self.content.read())
|
|
||||||
self.content.close()
|
|
||||||
|
|
||||||
|
|
||||||
class CSVZipBookWriter(BookWriter):
|
|
||||||
"""
|
|
||||||
csvz writer
|
|
||||||
|
|
||||||
It is better to store csv files as a csvz as it saves your disk space.
|
|
||||||
Pyexcel-io had the facility to unzip it for you or you could use
|
|
||||||
any other unzip software.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
BookWriter.__init__(self)
|
|
||||||
self._file_type = FILE_FORMAT_CSVZ
|
|
||||||
self.zipfile = None
|
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
|
||||||
BookWriter.open(self, file_name, **keywords)
|
|
||||||
self.zipfile = zipfile.ZipFile(file_name, "w", zipfile.ZIP_DEFLATED)
|
|
||||||
|
|
||||||
def create_sheet(self, name):
|
|
||||||
given_name = name
|
|
||||||
if given_name is None:
|
|
||||||
given_name = DEFAULT_SHEET_NAME
|
|
||||||
writer = CSVZipSheetWriter(
|
|
||||||
self.zipfile, given_name, self._file_type[:3], **self._keywords
|
|
||||||
)
|
|
||||||
return writer
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
self.zipfile.close()
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
"""
|
||||||
|
pyexcel_io.fileformat.csvz_sheet
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
The lower level csvz file format handler.
|
||||||
|
|
||||||
|
:copyright: (c) 2014-2022 by Onni Software Ltd.
|
||||||
|
:license: New BSD License, see LICENSE for more details
|
||||||
|
"""
|
||||||
|
import csv
|
||||||
|
|
||||||
|
from pyexcel_io._compact import StringIO
|
||||||
|
from pyexcel_io.writers.csv_sheet import CSVFileWriter
|
||||||
|
|
||||||
|
|
||||||
|
class CSVZipSheetWriter(CSVFileWriter):
|
||||||
|
"""handle the zipfile interface"""
|
||||||
|
|
||||||
|
def __init__(self, zipfile, sheetname, file_extension, **keywords):
|
||||||
|
self.file_extension = file_extension
|
||||||
|
keywords["single_sheet_in_book"] = False
|
||||||
|
self.content = StringIO()
|
||||||
|
super().__init__(zipfile, sheetname, **keywords)
|
||||||
|
|
||||||
|
def get_writer(self):
|
||||||
|
return csv.writer(self.content, **self._keywords)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
file_name = "%s.%s" % (self._sheet_name, self.file_extension)
|
||||||
|
self.content.seek(0)
|
||||||
|
self._native_book.writestr(file_name, self.content.read())
|
||||||
|
self.content.close()
|
|
@ -0,0 +1,35 @@
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
from pyexcel_io import constants
|
||||||
|
from pyexcel_io.plugin_api import IWriter
|
||||||
|
from pyexcel_io.writers.csvz_sheet import CSVZipSheetWriter
|
||||||
|
|
||||||
|
|
||||||
|
class CsvZipWriter(IWriter):
|
||||||
|
"""
|
||||||
|
csvz writer
|
||||||
|
|
||||||
|
It is better to store csv files as a csvz as it saves your disk space.
|
||||||
|
Pyexcel-io had the facility to unzip it for you or you could use
|
||||||
|
any other unzip software.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, file_name, file_type, **keywords):
|
||||||
|
self._file_type = file_type
|
||||||
|
self.zipfile = zipfile.ZipFile(file_name, "w", zipfile.ZIP_DEFLATED)
|
||||||
|
self._keywords = keywords
|
||||||
|
if file_type == constants.FILE_FORMAT_TSVZ:
|
||||||
|
self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
||||||
|
|
||||||
|
def create_sheet(self, name):
|
||||||
|
given_name = name
|
||||||
|
if given_name is None:
|
||||||
|
given_name = constants.DEFAULT_SHEET_NAME
|
||||||
|
writer = CSVZipSheetWriter(
|
||||||
|
self.zipfile, given_name, self._file_type[:3], **self._keywords
|
||||||
|
)
|
||||||
|
return writer
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
if self.zipfile:
|
||||||
|
self.zipfile.close()
|
|
@ -1,23 +0,0 @@
|
||||||
"""
|
|
||||||
pyexcel_io.fileformat.tsv
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
The lower level tsv file format handler.
|
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
|
||||||
:license: New BSD License, see LICENSE for more details
|
|
||||||
"""
|
|
||||||
import pyexcel_io.constants as constants
|
|
||||||
from .csvw import CSVBookWriter
|
|
||||||
|
|
||||||
|
|
||||||
class TSVBookWriter(CSVBookWriter):
|
|
||||||
""" write tsv """
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
CSVBookWriter.__init__(self)
|
|
||||||
self._file_type = constants.FILE_FORMAT_TSV
|
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
|
||||||
keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
|
|
||||||
CSVBookWriter.open(self, file_name, **keywords)
|
|
|
@ -1,27 +0,0 @@
|
||||||
"""
|
|
||||||
pyexcel_io.fileformat.tsvz
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
The lower level tsvz file format handler.
|
|
||||||
|
|
||||||
:copyright: (c) 2014-2017 by Onni Software Ltd.
|
|
||||||
:license: New BSD License, see LICENSE for more details
|
|
||||||
"""
|
|
||||||
from pyexcel_io.constants import FILE_FORMAT_TSVZ, KEYWORD_TSV_DIALECT
|
|
||||||
|
|
||||||
from .csvz import CSVZipBookWriter
|
|
||||||
|
|
||||||
|
|
||||||
class TSVZipBookWriter(CSVZipBookWriter):
|
|
||||||
""" write zipped tsv file
|
|
||||||
|
|
||||||
It is similiar to CSVZipBookWriter, but support tab separated values
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
CSVZipBookWriter.__init__(self)
|
|
||||||
self._file_type = FILE_FORMAT_TSVZ
|
|
||||||
|
|
||||||
def open(self, file_name, **keywords):
|
|
||||||
keywords["dialect"] = KEYWORD_TSV_DIALECT
|
|
||||||
CSVZipBookWriter.open(self, file_name, **keywords)
|
|
|
@ -1,2 +1 @@
|
||||||
ordereddict;python_version<"2.7"
|
|
||||||
lml>=0.0.4
|
lml>=0.0.4
|
||||||
|
|
|
@ -1,5 +1 @@
|
||||||
.
|
|
||||||
https://github.com/chfw/lml/archive/master.zip
|
|
||||||
https://github.com/pyexcel/pyexcel/archive/master.zip
|
|
||||||
https://github.com/pyexcel/pyexcel-xls/archive/master.zip
|
|
||||||
|
|
||||||
|
|
129
setup.py
129
setup.py
|
@ -1,6 +1,9 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
# Template by pypi-mobans
|
"""
|
||||||
|
Template by pypi-mobans
|
||||||
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import codecs
|
import codecs
|
||||||
|
@ -22,84 +25,86 @@ PY33 = sys.version_info < (3, 4)
|
||||||
try:
|
try:
|
||||||
lc = locale.getlocale()
|
lc = locale.getlocale()
|
||||||
pf = platform.system()
|
pf = platform.system()
|
||||||
if pf != 'Windows' and lc == (None, None):
|
if pf != "Windows" and lc == (None, None):
|
||||||
locale.setlocale(locale.LC_ALL, 'C.UTF-8')
|
locale.setlocale(locale.LC_ALL, "C.UTF-8")
|
||||||
except (ValueError, UnicodeError, locale.Error):
|
except (ValueError, UnicodeError, locale.Error):
|
||||||
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
|
locale.setlocale(locale.LC_ALL, "en_US.UTF-8")
|
||||||
|
|
||||||
NAME = 'pyexcel-io'
|
NAME = "pyexcel-io"
|
||||||
AUTHOR = 'C.W.'
|
AUTHOR = "C.W."
|
||||||
VERSION = '0.5.13'
|
VERSION = "0.6.6"
|
||||||
EMAIL = 'wangc_2011@hotmail.com'
|
EMAIL = "info@pyexcel.org"
|
||||||
LICENSE = 'New BSD'
|
LICENSE = "New BSD"
|
||||||
DESCRIPTION = (
|
DESCRIPTION = (
|
||||||
'A python library to read and write structured data in csv, zipped csv' +
|
"A python library to read and write structured data in csv, zipped csv" +
|
||||||
'format and to/from databases'
|
"format and to/from databases"
|
||||||
)
|
)
|
||||||
URL = 'https://github.com/pyexcel/pyexcel-io'
|
URL = "https://github.com/pyexcel/pyexcel-io"
|
||||||
DOWNLOAD_URL = '%s/archive/0.5.13.tar.gz' % URL
|
DOWNLOAD_URL = "%s/archive/0.6.6.tar.gz" % URL
|
||||||
FILES = ['README.rst', 'CHANGELOG.rst']
|
FILES = ["README.rst", "CHANGELOG.rst"]
|
||||||
KEYWORDS = [
|
KEYWORDS = [
|
||||||
'python',
|
"python",
|
||||||
'API',
|
"API",
|
||||||
'tsv',
|
"tsv",
|
||||||
'tsvz',
|
"tsvz",
|
||||||
'csv',
|
"csv",
|
||||||
'csvz',
|
"csvz",
|
||||||
'django',
|
"django",
|
||||||
'sqlalchemy',
|
"sqlalchemy",
|
||||||
]
|
]
|
||||||
|
|
||||||
CLASSIFIERS = [
|
CLASSIFIERS = [
|
||||||
'Topic :: Software Development :: Libraries',
|
"Topic :: Software Development :: Libraries",
|
||||||
'Programming Language :: Python',
|
"Programming Language :: Python",
|
||||||
'Intended Audience :: Developers',
|
"Intended Audience :: Developers",
|
||||||
'Programming Language :: Python :: 2.6',
|
|
||||||
'Programming Language :: Python :: 2.7',
|
"Programming Language :: Python :: 3 :: Only",
|
||||||
'Programming Language :: Python :: 3.3',
|
|
||||||
'Programming Language :: Python :: 3.4',
|
|
||||||
'Programming Language :: Python :: 3.5',
|
|
||||||
'Programming Language :: Python :: 3.6',
|
"Programming Language :: Python :: 3.6",
|
||||||
|
"Programming Language :: Python :: 3.7",
|
||||||
|
"Programming Language :: Python :: 3.8",
|
||||||
|
|
||||||
'License :: OSI Approved :: BSD License',
|
'License :: OSI Approved :: BSD License',
|
||||||
'Programming Language :: Python :: Implementation :: PyPy'
|
'Programming Language :: Python :: Implementation :: PyPy'
|
||||||
]
|
]
|
||||||
|
|
||||||
|
PYTHON_REQUIRES = ">=3.6"
|
||||||
|
|
||||||
INSTALL_REQUIRES = [
|
INSTALL_REQUIRES = [
|
||||||
'lml>=0.0.4',
|
"lml>=0.0.4",
|
||||||
]
|
]
|
||||||
SETUP_COMMANDS = {}
|
SETUP_COMMANDS = {}
|
||||||
|
|
||||||
if PY26:
|
PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests", "tests.*"])
|
||||||
INSTALL_REQUIRES.append('ordereddict')
|
|
||||||
|
|
||||||
PACKAGES = find_packages(exclude=['ez_setup', 'examples', 'tests'])
|
|
||||||
EXTRAS_REQUIRE = {
|
EXTRAS_REQUIRE = {
|
||||||
'xls': ['pyexcel-xls>=0.5.0'],
|
"xls": ['pyexcel-xls>=0.6.0'],
|
||||||
'xlsx': ['pyexcel-xlsx>=0.5.0'],
|
"xlsx": ['pyexcel-xlsx>=0.6.0'],
|
||||||
'ods': ['pyexcel-ods3>=0.5.0'],
|
"ods": ['pyexcel-ods3>=0.6.0'],
|
||||||
}
|
}
|
||||||
# You do not need to read beyond this line
|
# You do not need to read beyond this line
|
||||||
PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format(
|
PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable)
|
||||||
sys.executable)
|
HERE = os.path.abspath(os.path.dirname(__file__))
|
||||||
GS_COMMAND = ('gs pyexcel-io v0.5.13 ' +
|
|
||||||
"Find 0.5.13 in changelog for more details")
|
GS_COMMAND = ("gease pyexcel-io v0.6.6 " +
|
||||||
NO_GS_MESSAGE = ('Automatic github release is disabled. ' +
|
"Find 0.6.6 in changelog for more details")
|
||||||
'Please install gease to enable it.')
|
NO_GS_MESSAGE = ("Automatic github release is disabled. " +
|
||||||
|
"Please install gease to enable it.")
|
||||||
UPLOAD_FAILED_MSG = (
|
UPLOAD_FAILED_MSG = (
|
||||||
'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND)
|
'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND)
|
||||||
HERE = os.path.abspath(os.path.dirname(__file__))
|
|
||||||
|
|
||||||
|
|
||||||
class PublishCommand(Command):
|
class PublishCommand(Command):
|
||||||
"""Support setup.py upload."""
|
"""Support setup.py upload."""
|
||||||
|
|
||||||
description = 'Build and publish the package on github and pypi'
|
description = "Build and publish the package on github and pypi"
|
||||||
user_options = []
|
user_options = []
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def status(s):
|
def status(s):
|
||||||
"""Prints things in bold."""
|
"""Prints things in bold."""
|
||||||
print('\033[1m{0}\033[0m'.format(s))
|
print("\033[1m{0}\033[0m".format(s))
|
||||||
|
|
||||||
def initialize_options(self):
|
def initialize_options(self):
|
||||||
pass
|
pass
|
||||||
|
@ -109,14 +114,14 @@ class PublishCommand(Command):
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
try:
|
try:
|
||||||
self.status('Removing previous builds...')
|
self.status("Removing previous builds...")
|
||||||
rmtree(os.path.join(HERE, 'dist'))
|
rmtree(os.path.join(HERE, "dist"))
|
||||||
rmtree(os.path.join(HERE, 'build'))
|
rmtree(os.path.join(HERE, "build"))
|
||||||
rmtree(os.path.join(HERE, 'pyexcel_io.egg-info'))
|
rmtree(os.path.join(HERE, "pyexcel_io.egg-info"))
|
||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
self.status('Building Source and Wheel (universal) distribution...')
|
self.status("Building Source and Wheel (universal) distribution...")
|
||||||
run_status = True
|
run_status = True
|
||||||
if has_gease():
|
if has_gease():
|
||||||
run_status = os.system(GS_COMMAND) == 0
|
run_status = os.system(GS_COMMAND) == 0
|
||||||
|
@ -124,16 +129,15 @@ class PublishCommand(Command):
|
||||||
self.status(NO_GS_MESSAGE)
|
self.status(NO_GS_MESSAGE)
|
||||||
if run_status:
|
if run_status:
|
||||||
if os.system(PUBLISH_COMMAND) != 0:
|
if os.system(PUBLISH_COMMAND) != 0:
|
||||||
self.status(UPLOAD_FAILED_MSG % PUBLISH_COMMAND)
|
self.status(UPLOAD_FAILED_MSG)
|
||||||
|
|
||||||
sys.exit()
|
sys.exit()
|
||||||
|
|
||||||
|
|
||||||
SETUP_COMMANDS.update({
|
SETUP_COMMANDS.update({
|
||||||
'publish': PublishCommand
|
"publish": PublishCommand
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
def has_gease():
|
def has_gease():
|
||||||
"""
|
"""
|
||||||
test if github release command is installed
|
test if github release command is installed
|
||||||
|
@ -159,7 +163,7 @@ def read_files(*files):
|
||||||
def read(afile):
|
def read(afile):
|
||||||
"""Read a file into setup"""
|
"""Read a file into setup"""
|
||||||
the_relative_file = os.path.join(HERE, afile)
|
the_relative_file = os.path.join(HERE, afile)
|
||||||
with codecs.open(the_relative_file, 'r', 'utf-8') as opened_file:
|
with codecs.open(the_relative_file, "r", "utf-8") as opened_file:
|
||||||
content = filter_out_test_code(opened_file)
|
content = filter_out_test_code(opened_file)
|
||||||
content = "".join(list(content))
|
content = "".join(list(content))
|
||||||
return content
|
return content
|
||||||
|
@ -168,11 +172,11 @@ def read(afile):
|
||||||
def filter_out_test_code(file_handle):
|
def filter_out_test_code(file_handle):
|
||||||
found_test_code = False
|
found_test_code = False
|
||||||
for line in file_handle.readlines():
|
for line in file_handle.readlines():
|
||||||
if line.startswith('.. testcode:'):
|
if line.startswith(".. testcode:"):
|
||||||
found_test_code = True
|
found_test_code = True
|
||||||
continue
|
continue
|
||||||
if found_test_code is True:
|
if found_test_code is True:
|
||||||
if line.startswith(' '):
|
if line.startswith(" "):
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
empty_line = line.strip()
|
empty_line = line.strip()
|
||||||
|
@ -182,14 +186,14 @@ def filter_out_test_code(file_handle):
|
||||||
found_test_code = False
|
found_test_code = False
|
||||||
yield line
|
yield line
|
||||||
else:
|
else:
|
||||||
for keyword in ['|version|', '|today|']:
|
for keyword in ["|version|", "|today|"]:
|
||||||
if keyword in line:
|
if keyword in line:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
yield line
|
yield line
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == "__main__":
|
||||||
setup(
|
setup(
|
||||||
test_suite="tests",
|
test_suite="tests",
|
||||||
name=NAME,
|
name=NAME,
|
||||||
|
@ -202,8 +206,9 @@ if __name__ == '__main__':
|
||||||
long_description=read_files(*FILES),
|
long_description=read_files(*FILES),
|
||||||
license=LICENSE,
|
license=LICENSE,
|
||||||
keywords=KEYWORDS,
|
keywords=KEYWORDS,
|
||||||
|
python_requires=PYTHON_REQUIRES,
|
||||||
extras_require=EXTRAS_REQUIRE,
|
extras_require=EXTRAS_REQUIRE,
|
||||||
tests_require=['nose'],
|
tests_require=["nose"],
|
||||||
install_requires=INSTALL_REQUIRES,
|
install_requires=INSTALL_REQUIRES,
|
||||||
packages=PACKAGES,
|
packages=PACKAGES,
|
||||||
include_package_data=True,
|
include_package_data=True,
|
||||||
|
|
2
test.bat
2
test.bat
|
@ -1,2 +1,2 @@
|
||||||
pip freeze
|
pip freeze
|
||||||
nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io && flake8 . --exclude=.moban.d,docs --builtins=unicode,xrange,long
|
nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io
|
||||||
|
|
3
test.sh
3
test.sh
|
@ -1,2 +1,3 @@
|
||||||
|
#/bin/bash
|
||||||
pip freeze
|
pip freeze
|
||||||
nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io && flake8 . --exclude=.moban.d,docs --builtins=unicode,xrange,long
|
nosetests --with-coverage --cover-package pyexcel_io --cover-package tests tests --with-doctest --doctest-extension=.rst README.rst docs/source pyexcel_io
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
# needed for Python 2.7, python setup.py test to discover tests directory
|
|
@ -3,6 +3,13 @@ mock;python_version<"3"
|
||||||
codecov
|
codecov
|
||||||
coverage
|
coverage
|
||||||
flake8
|
flake8
|
||||||
|
black
|
||||||
|
isort
|
||||||
|
collective.checkdocs
|
||||||
|
pygments
|
||||||
|
moban
|
||||||
|
moban_jinja2_github
|
||||||
|
pyexcel
|
||||||
|
pyexcel-xls==0.5.9
|
||||||
SQLAlchemy
|
SQLAlchemy
|
||||||
pyexcel>=0.2.0
|
pyexcel-xlsxw
|
||||||
pyexcel-xls>=0.1.0
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
from pyexcel_io.sheet import SheetReader, SheetWriter, NamedContent
|
|
||||||
from pyexcel_io.book import BookWriter
|
from pyexcel_io.book import BookWriter
|
||||||
|
from pyexcel_io.sheet import SheetReader, SheetWriter, NamedContent
|
||||||
from pyexcel_io.utils import is_empty_array
|
from pyexcel_io.utils import is_empty_array
|
||||||
|
|
||||||
from nose.tools import raises
|
from nose.tools import raises
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,12 @@
|
||||||
|
from pyexcel_io.book import (
|
||||||
|
BookReader,
|
||||||
|
BookWriter,
|
||||||
|
RWInterface,
|
||||||
|
_convert_content_to_stream,
|
||||||
|
)
|
||||||
|
from pyexcel_io._compact import BytesIO, StringIO
|
||||||
|
|
||||||
from nose.tools import raises
|
from nose.tools import raises
|
||||||
from pyexcel_io.book import RWInterface, BookReader, BookWriter
|
|
||||||
|
|
||||||
|
|
||||||
@raises(NotImplementedError)
|
@raises(NotImplementedError)
|
||||||
|
@ -30,3 +37,15 @@ def test_book_reader_open_stream():
|
||||||
def test_book_writer():
|
def test_book_writer():
|
||||||
writer = BookWriter()
|
writer = BookWriter()
|
||||||
writer.open_stream("a string")
|
writer.open_stream("a string")
|
||||||
|
|
||||||
|
|
||||||
|
def test_convert_to_bytes_stream():
|
||||||
|
file_content = b"test"
|
||||||
|
stream = _convert_content_to_stream(file_content, "csv")
|
||||||
|
assert isinstance(stream, StringIO)
|
||||||
|
|
||||||
|
|
||||||
|
def test_convert_to_string_stream():
|
||||||
|
file_content = "test"
|
||||||
|
stream = _convert_content_to_stream(file_content, "csvz")
|
||||||
|
assert isinstance(stream, BytesIO)
|
||||||
|
|
|
@ -1,19 +1,20 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from unittest import TestCase
|
|
||||||
from textwrap import dedent
|
from textwrap import dedent
|
||||||
from nose.tools import raises, eq_
|
from unittest import TestCase
|
||||||
|
|
||||||
import pyexcel_io.manager as manager
|
import pyexcel_io.manager as manager
|
||||||
|
from pyexcel_io import get_data
|
||||||
from pyexcel_io.sheet import NamedContent
|
from pyexcel_io.sheet import NamedContent
|
||||||
from pyexcel_io.readers.csvr import (
|
from pyexcel_io.reader import EncapsulatedSheetReader
|
||||||
CSVSheetReader,
|
from pyexcel_io._compact import BytesIO, StringIO
|
||||||
|
from pyexcel_io.readers.csv_sheet import (
|
||||||
CSVFileReader,
|
CSVFileReader,
|
||||||
|
CSVSheetReader,
|
||||||
CSVinMemoryReader,
|
CSVinMemoryReader,
|
||||||
)
|
)
|
||||||
from pyexcel_io.writers.csvw import CSVFileWriter, CSVMemoryWriter
|
from pyexcel_io.writers.csv_sheet import CSVFileWriter, CSVMemoryWriter
|
||||||
from pyexcel_io._compact import BytesIO, PY2, StringIO
|
|
||||||
|
from nose.tools import eq_, raises
|
||||||
|
|
||||||
|
|
||||||
class TestReaders(TestCase):
|
class TestReaders(TestCase):
|
||||||
|
@ -32,7 +33,9 @@ class TestReaders(TestCase):
|
||||||
sheet.get_file_handle()
|
sheet.get_file_handle()
|
||||||
|
|
||||||
def test_sheet_file_reader(self):
|
def test_sheet_file_reader(self):
|
||||||
r = CSVFileReader(NamedContent(self.file_type, self.test_file))
|
r = EncapsulatedSheetReader(
|
||||||
|
CSVFileReader(NamedContent(self.file_type, self.test_file))
|
||||||
|
)
|
||||||
result = list(r.to_array())
|
result = list(r.to_array())
|
||||||
self.assertEqual(result, self.expected_data)
|
self.assertEqual(result, self.expected_data)
|
||||||
|
|
||||||
|
@ -41,7 +44,10 @@ class TestReaders(TestCase):
|
||||||
with open(self.test_file, "r") as f:
|
with open(self.test_file, "r") as f:
|
||||||
io.write(f.read())
|
io.write(f.read())
|
||||||
io.seek(0)
|
io.seek(0)
|
||||||
r = CSVinMemoryReader(NamedContent(self.file_type, io))
|
r = EncapsulatedSheetReader(
|
||||||
|
CSVinMemoryReader(NamedContent(self.file_type, io))
|
||||||
|
)
|
||||||
|
|
||||||
result = list(r.to_array())
|
result = list(r.to_array())
|
||||||
self.assertEqual(result, self.expected_data)
|
self.assertEqual(result, self.expected_data)
|
||||||
|
|
||||||
|
@ -108,20 +114,37 @@ class TestNonUniformCSV(TestCase):
|
||||||
f.write(",".join(row) + "\n")
|
f.write(",".join(row) + "\n")
|
||||||
|
|
||||||
def test_sheet_file_reader(self):
|
def test_sheet_file_reader(self):
|
||||||
r = CSVFileReader(NamedContent(self.file_type, self.test_file))
|
r = EncapsulatedSheetReader(
|
||||||
|
CSVFileReader(NamedContent(self.file_type, self.test_file))
|
||||||
|
)
|
||||||
result = list(r.to_array())
|
result = list(r.to_array())
|
||||||
self.assertEqual(result, [[1], [4, 5, 6], ["", 7]])
|
self.assertEqual(result, [[1], [4, 5, 6], ["", 7]])
|
||||||
|
|
||||||
|
def test_sheet_file_reader_with_trailing_empty_cells(self):
|
||||||
|
r = EncapsulatedSheetReader(
|
||||||
|
CSVFileReader(NamedContent(self.file_type, self.test_file)),
|
||||||
|
keep_trailing_empty_cells=True,
|
||||||
|
)
|
||||||
|
result = list(r.to_array())
|
||||||
|
self.assertEqual(result, [[1], [4, 5, 6, "", ""], ["", 7]])
|
||||||
|
|
||||||
|
def test_get_data_with_trailing_empty_cells(self):
|
||||||
|
result = get_data(self.test_file, keep_trailing_empty_cells=True)
|
||||||
|
self.assertEqual(
|
||||||
|
result[self.test_file], [[1], [4, 5, 6, "", ""], ["", 7]]
|
||||||
|
)
|
||||||
|
|
||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
os.unlink(self.test_file)
|
os.unlink(self.test_file)
|
||||||
|
|
||||||
|
|
||||||
def test_utf16_decoding():
|
def test_utf16_decoding():
|
||||||
test_file = os.path.join("tests", "fixtures", "csv-encoding-utf16.csv")
|
test_file = os.path.join("tests", "fixtures", "csv-encoding-utf16.csv")
|
||||||
reader = CSVFileReader(NamedContent("csv", test_file), encoding="utf-16")
|
reader = EncapsulatedSheetReader(
|
||||||
|
CSVFileReader(NamedContent("csv", test_file), encoding="utf-16")
|
||||||
|
)
|
||||||
|
|
||||||
content = list(reader.to_array())
|
content = list(reader.to_array())
|
||||||
if PY2:
|
|
||||||
content[0] = [s.encode("utf-8") for s in content[0]]
|
|
||||||
expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]]
|
expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]]
|
||||||
eq_(content, expected)
|
eq_(content, expected)
|
||||||
|
|
||||||
|
@ -136,8 +159,6 @@ def test_utf16_encoding():
|
||||||
writer.close()
|
writer.close()
|
||||||
with open(test_file, "rb") as f:
|
with open(test_file, "rb") as f:
|
||||||
actual = f.read().decode("utf-16")
|
actual = f.read().decode("utf-16")
|
||||||
if PY2:
|
|
||||||
actual = actual.encode("utf-8")
|
|
||||||
eq_(actual, "Äkkilähdöt,Matkakirjoituksia,Matkatoimistot\n")
|
eq_(actual, "Äkkilähdöt,Matkakirjoituksia,Matkatoimistot\n")
|
||||||
os.unlink(test_file)
|
os.unlink(test_file)
|
||||||
|
|
||||||
|
@ -145,12 +166,11 @@ def test_utf16_encoding():
|
||||||
def test_utf16_memory_decoding():
|
def test_utf16_memory_decoding():
|
||||||
test_content = u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot"
|
test_content = u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot"
|
||||||
test_content = BytesIO(test_content.encode("utf-16"))
|
test_content = BytesIO(test_content.encode("utf-16"))
|
||||||
reader = CSVinMemoryReader(
|
reader = EncapsulatedSheetReader(
|
||||||
NamedContent("csv", test_content), encoding="utf-16"
|
CSVinMemoryReader(NamedContent("csv", test_content), encoding="utf-16")
|
||||||
)
|
)
|
||||||
|
|
||||||
content = list(reader.to_array())
|
content = list(reader.to_array())
|
||||||
if PY2:
|
|
||||||
content[0] = [s.encode("utf-8") for s in content[0]]
|
|
||||||
expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]]
|
expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]]
|
||||||
eq_(content, expected)
|
eq_(content, expected)
|
||||||
|
|
||||||
|
@ -167,6 +187,4 @@ def test_utf16_memory_encoding():
|
||||||
)
|
)
|
||||||
writer.write_array(content)
|
writer.write_array(content)
|
||||||
actual = io.getvalue()
|
actual = io.getvalue()
|
||||||
if PY2:
|
|
||||||
actual = actual.decode("utf-16")
|
|
||||||
eq_(actual, u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot\n")
|
eq_(actual, u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot\n")
|
||||||
|
|
|
@ -1,21 +1,23 @@
|
||||||
from nose.tools import raises, eq_
|
|
||||||
from pyexcel_io import save_data
|
from pyexcel_io import save_data
|
||||||
|
from pyexcel_io.reader import EncapsulatedSheetReader
|
||||||
from pyexcel_io._compact import OrderedDict
|
from pyexcel_io._compact import OrderedDict
|
||||||
from pyexcel_io.constants import DB_DJANGO
|
from pyexcel_io.constants import DB_DJANGO
|
||||||
from pyexcel_io.database.common import (
|
from pyexcel_io.database.common import (
|
||||||
DjangoModelImporter,
|
|
||||||
DjangoModelImportAdapter,
|
|
||||||
DjangoModelExporter,
|
DjangoModelExporter,
|
||||||
|
DjangoModelImporter,
|
||||||
DjangoModelExportAdapter,
|
DjangoModelExportAdapter,
|
||||||
)
|
DjangoModelImportAdapter,
|
||||||
from pyexcel_io.database.importers.django import (
|
|
||||||
DjangoModelWriter,
|
|
||||||
DjangoBookWriter,
|
|
||||||
)
|
)
|
||||||
from pyexcel_io.database.exporters.django import (
|
from pyexcel_io.database.exporters.django import (
|
||||||
DjangoModelReader,
|
|
||||||
DjangoBookReader,
|
DjangoBookReader,
|
||||||
|
DjangoModelReader,
|
||||||
)
|
)
|
||||||
|
from pyexcel_io.database.importers.django import (
|
||||||
|
DjangoBookWriter,
|
||||||
|
DjangoModelWriter,
|
||||||
|
)
|
||||||
|
|
||||||
|
from nose.tools import eq_, raises
|
||||||
|
|
||||||
|
|
||||||
class Package:
|
class Package:
|
||||||
|
@ -156,10 +158,13 @@ class TestSheet:
|
||||||
writer = DjangoModelWriter(None, adapter)
|
writer = DjangoModelWriter(None, adapter)
|
||||||
writer.write_array(self.data[1:])
|
writer.write_array(self.data[1:])
|
||||||
writer.close()
|
writer.close()
|
||||||
assert model.objects.objs == [
|
eq_(
|
||||||
{"Y": 2, "X": 2, "Z": 3},
|
model.objects.objs,
|
||||||
{"Y": 5, "X": 5, "Z": 6},
|
[
|
||||||
]
|
{"Y": 2, "X": 2, "Z": 3},
|
||||||
|
{"Y": 5, "X": 5, "Z": 6},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
def test_sheet_save_to_django_model_skip_me(self):
|
def test_sheet_save_to_django_model_skip_me(self):
|
||||||
model = FakeDjangoModel()
|
model = FakeDjangoModel()
|
||||||
|
@ -176,7 +181,7 @@ class TestSheet:
|
||||||
writer = DjangoModelWriter(None, adapter)
|
writer = DjangoModelWriter(None, adapter)
|
||||||
writer.write_array(self.data[1:])
|
writer.write_array(self.data[1:])
|
||||||
writer.close()
|
writer.close()
|
||||||
assert model.objects.objs == [{"Y": 2, "X": 1, "Z": 3}]
|
eq_(model.objects.objs, [{"Y": 2, "X": 1, "Z": 3}])
|
||||||
|
|
||||||
def test_load_sheet_from_django_model(self):
|
def test_load_sheet_from_django_model(self):
|
||||||
model = FakeDjangoModel()
|
model = FakeDjangoModel()
|
||||||
|
@ -209,7 +214,9 @@ class TestSheet:
|
||||||
return [str(element) for element in row]
|
return [str(element) for element in row]
|
||||||
|
|
||||||
# the key point of this test case
|
# the key point of this test case
|
||||||
reader = DjangoModelReader(model, row_renderer=row_renderer)
|
reader = EncapsulatedSheetReader(
|
||||||
|
DjangoModelReader(model), row_renderer=row_renderer
|
||||||
|
)
|
||||||
data = reader.to_array()
|
data = reader.to_array()
|
||||||
expected = [["X", "Y", "Z"], ["1", "2", "3"], ["4", "5", "6"]]
|
expected = [["X", "Y", "Z"], ["1", "2", "3"], ["4", "5", "6"]]
|
||||||
eq_(list(data), expected)
|
eq_(list(data), expected)
|
||||||
|
@ -237,6 +244,18 @@ class TestSheet:
|
||||||
writer.close()
|
writer.close()
|
||||||
eq_(model.objects.objs, self.result)
|
eq_(model.objects.objs, self.result)
|
||||||
|
|
||||||
|
def test_jumping_columns(self):
|
||||||
|
data2 = [["D", "A", "B", "C"], [1, 1, 2, 3], [10, 4, 5, 6]]
|
||||||
|
mapdict = {"C": "Z", "A": "X", "B": "Y"}
|
||||||
|
model = FakeDjangoModel()
|
||||||
|
adapter = DjangoModelImportAdapter(model)
|
||||||
|
adapter.column_names = data2[0]
|
||||||
|
adapter.column_name_mapping_dict = mapdict
|
||||||
|
writer = DjangoModelWriter(None, adapter)
|
||||||
|
writer.write_array(data2[1:])
|
||||||
|
writer.close()
|
||||||
|
eq_(model.objects.objs, self.result)
|
||||||
|
|
||||||
def test_empty_model(self):
|
def test_empty_model(self):
|
||||||
model = FakeDjangoModel()
|
model = FakeDjangoModel()
|
||||||
reader = DjangoModelReader(model)
|
reader = DjangoModelReader(model)
|
||||||
|
@ -279,8 +298,7 @@ class TestMultipleModels:
|
||||||
adapter1.get_name(): self.content["Sheet1"][1:],
|
adapter1.get_name(): self.content["Sheet1"][1:],
|
||||||
adapter2.get_name(): self.content["Sheet2"][1:],
|
adapter2.get_name(): self.content["Sheet2"][1:],
|
||||||
}
|
}
|
||||||
writer = DjangoBookWriter()
|
writer = DjangoBookWriter(importer, "django", batch_size=sample_size)
|
||||||
writer.open_content(importer, batch_size=sample_size)
|
|
||||||
writer.write(to_store)
|
writer.write(to_store)
|
||||||
writer.close()
|
writer.close()
|
||||||
assert model1.objects.objs == self.result1
|
assert model1.objects.objs == self.result1
|
||||||
|
@ -302,8 +320,9 @@ class TestMultipleModels:
|
||||||
adapter1.get_name(): self.content["Sheet1"][1:],
|
adapter1.get_name(): self.content["Sheet1"][1:],
|
||||||
adapter2.get_name(): self.content["Sheet2"][1:],
|
adapter2.get_name(): self.content["Sheet2"][1:],
|
||||||
}
|
}
|
||||||
writer = DjangoBookWriter()
|
writer = DjangoBookWriter(
|
||||||
writer.open_content(importer, batch_size=sample_size, bulk_save=False)
|
importer, "django", batch_size=sample_size, bulk_save=False
|
||||||
|
)
|
||||||
writer.write(to_store)
|
writer.write(to_store)
|
||||||
writer.close()
|
writer.close()
|
||||||
assert model1.objects.objs == []
|
assert model1.objects.objs == []
|
||||||
|
@ -334,12 +353,11 @@ class TestMultipleModels:
|
||||||
adapter2 = DjangoModelExportAdapter(model2)
|
adapter2 = DjangoModelExportAdapter(model2)
|
||||||
exporter.append(adapter1)
|
exporter.append(adapter1)
|
||||||
exporter.append(adapter2)
|
exporter.append(adapter2)
|
||||||
reader = DjangoBookReader()
|
reader = DjangoBookReader(exporter, "django")
|
||||||
reader.open_content(exporter)
|
result = read_all(reader)
|
||||||
data = reader.read_all()
|
for key in result:
|
||||||
for key in data.keys():
|
result[key] = list(result[key])
|
||||||
data[key] = list(data[key])
|
eq_(result, self.content)
|
||||||
assert data == self.content
|
|
||||||
|
|
||||||
@raises(Exception)
|
@raises(Exception)
|
||||||
def test_special_case_where_only_one_model_used(self):
|
def test_special_case_where_only_one_model_used(self):
|
||||||
|
@ -353,28 +371,6 @@ class TestMultipleModels:
|
||||||
"Sheet2": self.content["Sheet2"][1:],
|
"Sheet2": self.content["Sheet2"][1:],
|
||||||
}
|
}
|
||||||
save_data(importer, to_store, file_type=DB_DJANGO)
|
save_data(importer, to_store, file_type=DB_DJANGO)
|
||||||
assert model1.objects.objs == self.result1
|
|
||||||
model1._meta.model_name = "Sheet1"
|
|
||||||
model1._meta.update(["X", "Y", "Z"])
|
|
||||||
exporter = DjangoModelExporter()
|
|
||||||
adapter = DjangoModelExportAdapter(model1)
|
|
||||||
exporter.append(adapter)
|
|
||||||
reader = DjangoBookReader()
|
|
||||||
reader.open_content(exporter)
|
|
||||||
data = reader.read_all()
|
|
||||||
assert list(data["Sheet1"]) == self.content["Sheet1"]
|
|
||||||
|
|
||||||
|
|
||||||
@raises(TypeError)
|
|
||||||
def test_not_implemented_method():
|
|
||||||
reader = DjangoBookReader()
|
|
||||||
reader.open("afile")
|
|
||||||
|
|
||||||
|
|
||||||
@raises(TypeError)
|
|
||||||
def test_not_implemented_method_2():
|
|
||||||
reader = DjangoBookReader()
|
|
||||||
reader.open_stream("afile")
|
|
||||||
|
|
||||||
|
|
||||||
class TestFilter:
|
class TestFilter:
|
||||||
|
@ -393,25 +389,33 @@ class TestFilter:
|
||||||
self.model._meta.update(["X", "Y", "Z"])
|
self.model._meta.update(["X", "Y", "Z"])
|
||||||
|
|
||||||
def test_load_sheet_from_django_model_with_filter(self):
|
def test_load_sheet_from_django_model_with_filter(self):
|
||||||
reader = DjangoModelReader(self.model, start_row=0, row_limit=2)
|
reader = EncapsulatedSheetReader(
|
||||||
|
DjangoModelReader(self.model), start_row=0, row_limit=2
|
||||||
|
)
|
||||||
data = reader.to_array()
|
data = reader.to_array()
|
||||||
expected = [["X", "Y", "Z"], [1, 2, 3]]
|
expected = [["X", "Y", "Z"], [1, 2, 3]]
|
||||||
eq_(list(data), expected)
|
eq_(list(data), expected)
|
||||||
|
|
||||||
def test_load_sheet_from_django_model_with_filter_1(self):
|
def test_load_sheet_from_django_model_with_filter_1(self):
|
||||||
reader = DjangoModelReader(self.model, start_row=1, row_limit=3)
|
reader = EncapsulatedSheetReader(
|
||||||
|
DjangoModelReader(self.model), start_row=1, row_limit=3
|
||||||
|
)
|
||||||
data = reader.to_array()
|
data = reader.to_array()
|
||||||
expected = [[1, 2, 3], [4, 5, 6]]
|
expected = [[1, 2, 3], [4, 5, 6]]
|
||||||
eq_(list(data), expected)
|
eq_(list(data), expected)
|
||||||
|
|
||||||
def test_load_sheet_from_django_model_with_filter_2(self):
|
def test_load_sheet_from_django_model_with_filter_2(self):
|
||||||
reader = DjangoModelReader(self.model, start_column=1)
|
reader = EncapsulatedSheetReader(
|
||||||
|
DjangoModelReader(self.model), start_column=1
|
||||||
|
)
|
||||||
data = reader.to_array()
|
data = reader.to_array()
|
||||||
expected = [["Y", "Z"], [2, 3], [5, 6]]
|
expected = [["Y", "Z"], [2, 3], [5, 6]]
|
||||||
eq_(list(data), expected)
|
eq_(list(data), expected)
|
||||||
|
|
||||||
def test_load_sheet_from_django_model_with_filter_3(self):
|
def test_load_sheet_from_django_model_with_filter_3(self):
|
||||||
reader = DjangoModelReader(self.model, start_column=1, column_limit=1)
|
reader = EncapsulatedSheetReader(
|
||||||
|
DjangoModelReader(self.model), start_column=1, column_limit=1
|
||||||
|
)
|
||||||
data = reader.to_array()
|
data = reader.to_array()
|
||||||
expected = [["Y"], [2], [5]]
|
expected = [["Y"], [2], [5]]
|
||||||
eq_(list(data), expected)
|
eq_(list(data), expected)
|
||||||
|
@ -422,3 +426,10 @@ def test_django_model_import_adapter():
|
||||||
adapter.column_names = ["a"]
|
adapter.column_names = ["a"]
|
||||||
adapter.row_initializer = "abc"
|
adapter.row_initializer = "abc"
|
||||||
eq_(adapter.row_initializer, "abc")
|
eq_(adapter.row_initializer, "abc")
|
||||||
|
|
||||||
|
|
||||||
|
def read_all(reader):
|
||||||
|
result = OrderedDict()
|
||||||
|
for index, sheet in enumerate(reader.content_array):
|
||||||
|
result.update({sheet.name: reader.read_sheet(index).to_array()})
|
||||||
|
return result
|
||||||
|
|
|
@ -1,9 +1,10 @@
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
import pyexcel_io.constants as constants
|
||||||
from pyexcel_io import get_data, save_data
|
from pyexcel_io import get_data, save_data
|
||||||
from pyexcel_io.utils import _index_filter
|
from pyexcel_io.utils import _index_filter
|
||||||
|
|
||||||
from nose.tools import eq_
|
from nose.tools import eq_
|
||||||
import pyexcel_io.constants as constants
|
|
||||||
|
|
||||||
|
|
||||||
def test_index_filter():
|
def test_index_filter():
|
||||||
|
|
|
@ -1,18 +1,20 @@
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
import types
|
import types
|
||||||
|
from zipfile import BadZipfile
|
||||||
from unittest import TestCase
|
from unittest import TestCase
|
||||||
|
|
||||||
import pyexcel_io.manager as manager
|
import pyexcel_io.manager as manager
|
||||||
import pyexcel_io.exceptions as exceptions
|
import pyexcel_io.exceptions as exceptions
|
||||||
from pyexcel_io._compact import StringIO, BytesIO, is_string
|
from pyexcel_io import get_data, iget_data, save_data
|
||||||
from pyexcel_io._compact import OrderedDict
|
|
||||||
from pyexcel_io import save_data, get_data, iget_data
|
|
||||||
from pyexcel_io.io import load_data, get_writer
|
from pyexcel_io.io import load_data, get_writer
|
||||||
from nose.tools import raises, eq_
|
from pyexcel_io._compact import BytesIO, StringIO, OrderedDict, is_string
|
||||||
from zipfile import BadZipfile
|
|
||||||
|
from nose.tools import eq_, raises
|
||||||
|
|
||||||
|
|
||||||
PY2 = sys.version_info[0] == 2
|
@raises(IOError)
|
||||||
|
def test_directory_name_as_file():
|
||||||
|
get_data("/")
|
||||||
|
|
||||||
|
|
||||||
def test_force_file_type():
|
def test_force_file_type():
|
||||||
|
@ -24,6 +26,20 @@ def test_force_file_type():
|
||||||
eq_(expected, data[test_file])
|
eq_(expected, data[test_file])
|
||||||
|
|
||||||
|
|
||||||
|
def test_force_file_type_on_write():
|
||||||
|
test_file = "force_file_type_on_write.txt"
|
||||||
|
save_data(test_file, {"sheet 1": [[1, 2]]}, force_file_type="csv")
|
||||||
|
data = get_data(test_file, force_file_type="csv")
|
||||||
|
expected = [[1, 2]]
|
||||||
|
eq_(expected, data[test_file])
|
||||||
|
os.unlink(test_file)
|
||||||
|
|
||||||
|
|
||||||
|
@raises(IOError)
|
||||||
|
def test_invalid_file():
|
||||||
|
load_data("/something/does/not/exist")
|
||||||
|
|
||||||
|
|
||||||
@raises(IOError)
|
@raises(IOError)
|
||||||
def test_no_valid_parameters():
|
def test_no_valid_parameters():
|
||||||
load_data()
|
load_data()
|
||||||
|
@ -49,14 +65,14 @@ def test_wrong_parameter_to_get_writer():
|
||||||
get_writer(1)
|
get_writer(1)
|
||||||
|
|
||||||
|
|
||||||
@raises(Exception)
|
# @raises(Exception)
|
||||||
def test_wrong_parameter_to_get_writer2():
|
# def test_wrong_parameter_to_get_writer2():
|
||||||
get_writer(1, file_type="csv")
|
# get_writer(1, file_type="csv")
|
||||||
|
|
||||||
|
|
||||||
def test_load_ods_data():
|
def test_load_ods_data():
|
||||||
msg = "Please install one of these plugins for read data in 'ods': "
|
msg = "Please install one of these plugins for read data in 'ods': "
|
||||||
msg += "pyexcel-ods,pyexcel-ods3"
|
msg += "pyexcel-ods,pyexcel-ods3,pyexcel-odsr"
|
||||||
try:
|
try:
|
||||||
get_data("test.ods")
|
get_data("test.ods")
|
||||||
except exceptions.SupportingPluginAvailableButNotInstalled as e:
|
except exceptions.SupportingPluginAvailableButNotInstalled as e:
|
||||||
|
@ -66,7 +82,7 @@ def test_load_ods_data():
|
||||||
def test_load_ods_data_from_memory():
|
def test_load_ods_data_from_memory():
|
||||||
io = BytesIO()
|
io = BytesIO()
|
||||||
msg = "Please install one of these plugins for read data in 'ods': "
|
msg = "Please install one of these plugins for read data in 'ods': "
|
||||||
msg += "pyexcel-ods,pyexcel-ods3"
|
msg += "pyexcel-ods,pyexcel-ods3,pyexcel-odsr"
|
||||||
try:
|
try:
|
||||||
get_data(io, file_type="ods")
|
get_data(io, file_type="ods")
|
||||||
except exceptions.SupportingPluginAvailableButNotInstalled as e:
|
except exceptions.SupportingPluginAvailableButNotInstalled as e:
|
||||||
|
@ -84,7 +100,7 @@ def test_write_xlsx_data_to_memory():
|
||||||
eq_(str(e), msg)
|
eq_(str(e), msg)
|
||||||
|
|
||||||
|
|
||||||
@raises(exceptions.NoSupportingPluginFound)
|
@raises(IOError)
|
||||||
def test_load_unknown_data():
|
def test_load_unknown_data():
|
||||||
get_data("test.unknown")
|
get_data("test.unknown")
|
||||||
|
|
||||||
|
@ -97,11 +113,8 @@ def test_load_unknown_data_from_memory():
|
||||||
|
|
||||||
@raises(BadZipfile)
|
@raises(BadZipfile)
|
||||||
def test_load_csvz_data_from_memory():
|
def test_load_csvz_data_from_memory():
|
||||||
if not PY2:
|
io = StringIO()
|
||||||
io = StringIO()
|
get_data(io, file_type="csvz")
|
||||||
get_data(io, file_type="csvz")
|
|
||||||
else:
|
|
||||||
raise BadZipfile("pass it")
|
|
||||||
|
|
||||||
|
|
||||||
@raises(IOError)
|
@raises(IOError)
|
||||||
|
@ -109,19 +122,11 @@ def test_write_xlsx_data():
|
||||||
get_data("test.xlsx")
|
get_data("test.xlsx")
|
||||||
|
|
||||||
|
|
||||||
@raises(exceptions.NoSupportingPluginFound)
|
|
||||||
def test_write_unknown_data():
|
|
||||||
get_data("test.unknown")
|
|
||||||
|
|
||||||
|
|
||||||
@raises(Exception)
|
@raises(Exception)
|
||||||
def test_writer_csvz_data_from_memory():
|
def test_writer_csvz_data_from_memory():
|
||||||
if not PY2:
|
io = StringIO()
|
||||||
io = StringIO()
|
writer = get_writer(io, file_type="csvz")
|
||||||
writer = get_writer(io, file_type="csvz")
|
writer.write({"adb": [[2, 3]]})
|
||||||
writer.write({"adb": [[2, 3]]})
|
|
||||||
else:
|
|
||||||
raise Exception("pass it")
|
|
||||||
|
|
||||||
|
|
||||||
@raises(exceptions.NoSupportingPluginFound)
|
@raises(exceptions.NoSupportingPluginFound)
|
||||||
|
@ -182,6 +187,7 @@ def test_file_handle_as_input():
|
||||||
with open(test_file, "r") as f:
|
with open(test_file, "r") as f:
|
||||||
data = get_data(f, "csv")
|
data = get_data(f, "csv")
|
||||||
eq_(data["csv"], [[1, 2, 3]])
|
eq_(data["csv"], [[1, 2, 3]])
|
||||||
|
os.unlink("file_handle.csv")
|
||||||
|
|
||||||
|
|
||||||
def test_file_type_case_insensitivity():
|
def test_file_type_case_insensitivity():
|
||||||
|
@ -192,6 +198,7 @@ def test_file_type_case_insensitivity():
|
||||||
with open(test_file, "r") as f:
|
with open(test_file, "r") as f:
|
||||||
data = get_data(f, "csv")
|
data = get_data(f, "csv")
|
||||||
eq_(data["csv"], [[1, 2, 3]])
|
eq_(data["csv"], [[1, 2, 3]])
|
||||||
|
os.unlink("file_handle.CSv")
|
||||||
|
|
||||||
|
|
||||||
def test_file_handle_as_output():
|
def test_file_handle_as_output():
|
||||||
|
@ -202,6 +209,7 @@ def test_file_handle_as_output():
|
||||||
with open(test_file, "r") as f:
|
with open(test_file, "r") as f:
|
||||||
content = f.read()
|
content = f.read()
|
||||||
eq_(content, "1,2,3\n")
|
eq_(content, "1,2,3\n")
|
||||||
|
os.unlink("file_handle.csv")
|
||||||
|
|
||||||
|
|
||||||
def test_binary_file_content():
|
def test_binary_file_content():
|
||||||
|
@ -247,10 +255,7 @@ def test_conversion_from_bytes_to_text():
|
||||||
|
|
||||||
|
|
||||||
def test_is_string():
|
def test_is_string():
|
||||||
if PY2:
|
assert is_string(type("a")) is True
|
||||||
assert is_string(type(u"a")) is True
|
|
||||||
else:
|
|
||||||
assert is_string(type("a")) is True
|
|
||||||
|
|
||||||
|
|
||||||
def test_generator_is_obtained():
|
def test_generator_is_obtained():
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue