add compatibility with python 3 (#25642)

This commit is contained in:
Frédéric Péters 2018-04-05 22:21:04 +02:00
parent 204722f788
commit 510bd0c84f
20 changed files with 83 additions and 73 deletions

View File

@ -33,7 +33,7 @@ class Base64FileField(fields.Field):
raise api_errors.APIError('NOT_BASE64')
max_size = self.get_max_size()
if max_size and len(content) > self.get_max_size():
raise api_errors.APIError('TOO_BIG', limit=self.get_max_size())
raise api_errors.APIError('TOO_BIG', limit=str(self.get_max_size()))
data = ContentFile(content, name=name)
else:
raise api_errors.APIError('NOT_STRING')

View File

@ -99,7 +99,7 @@ class PushDocumentSerializer(UserSerializerMixin):
user = data['user']
if (Document.objects.used_space(user) + data['file_b64_content'].size
> settings.FARGO_MAX_DOCUMENT_BOX_SIZE):
raise api_errors.APIError('BOX_IS_FULL', limit=settings.FARGO_MAX_DOCUMENT_BOX_SIZE)
raise api_errors.APIError('BOX_IS_FULL', limit=str(settings.FARGO_MAX_DOCUMENT_BOX_SIZE))
return data

View File

@ -12,7 +12,7 @@ import jsonfield.fields
class Migration(migrations.Migration):
replaces = [(b'fargo', '0001_initial'), (b'fargo', '0002_auto_20150818_2117'), (b'fargo', '0003_auto_20150924_1056'), (b'fargo', '0004_auto_20160212_0936'), (b'fargo', '0005_auto_20160312_1809'), (b'fargo', '0006_fill_new_columns'), (b'fargo', '0007_auto_20160312_1816'), (b'fargo', '0008_validation_origin'), (b'fargo', '0009_auto_20160326_2104'), (b'fargo', '0010_auto_20160413_0809'), (b'fargo', '0011_userdocument_deletable_by_user'), (b'fargo', '0012_auto_20161124_0626'), (b'fargo', '0013_document_mime_type'), (b'fargo', '0014_auto_20171016_0854'), (b'fargo', '0015_document_creation_date'), (b'fargo', '0016_auto_20180330_2248'), (b'fargo', '0017_auto_20180331_1532')]
replaces = [('fargo', '0001_initial'), ('fargo', '0002_auto_20150818_2117'), ('fargo', '0003_auto_20150924_1056'), ('fargo', '0004_auto_20160212_0936'), ('fargo', '0005_auto_20160312_1809'), ('fargo', '0006_fill_new_columns'), ('fargo', '0007_auto_20160312_1816'), ('fargo', '0008_validation_origin'), ('fargo', '0009_auto_20160326_2104'), ('fargo', '0010_auto_20160413_0809'), ('fargo', '0011_userdocument_deletable_by_user'), ('fargo', '0012_auto_20161124_0626'), ('fargo', '0013_document_mime_type'), ('fargo', '0014_auto_20171016_0854'), ('fargo', '0015_document_creation_date'), ('fargo', '0016_auto_20180330_2248'), ('fargo', '0017_auto_20180331_1532')]
initial = True

View File

@ -8,7 +8,9 @@ import threading
from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import force_text
from django.utils.text import slugify
from django.utils.http import urlquote
from django.utils.html import format_html
@ -24,6 +26,7 @@ from jsonfield import JSONField
from . import utils, managers
@python_2_unicode_compatible
class Origin(models.Model):
label = models.CharField(_('Label'), max_length=80)
slug = models.SlugField(_('Slug'))
@ -33,10 +36,11 @@ class Origin(models.Model):
self.slug = slugify(self.label)
return super(Origin, self).save(*args, **kwargs)
def __unicode__(self):
def __str__(self):
return self.label
@python_2_unicode_compatible
class UserDocument(models.Model):
'''Document uploaded by an user or an agent'''
user = models.ForeignKey(
@ -82,7 +86,7 @@ class UserDocument(models.Model):
def filename_encoded(self):
return urlquote(self.filename, safe='')
def __unicode__(self):
def __str__(self):
return self.title or self.filename
def get_download_url(self):
@ -105,6 +109,7 @@ class UserDocument(models.Model):
re.sub('[/\.+-]', '-', self.document.mime_type))
@python_2_unicode_compatible
class Validation(models.Model):
'''Validation of a document as special kind for an user,
the data field contains metadata extracted from the document.
@ -140,7 +145,7 @@ class Validation(models.Model):
template = self.document_type_schema.get('display_template', '')
if template:
try:
return unicode(template.format(**self.data))
return force_text(template.format(**self.data))
except KeyError:
pass
l = []
@ -150,7 +155,7 @@ class Validation(models.Model):
'label': meta_field['label'],
'value': self.data.get(meta_field['varname'], ''),
})
return unicode(u'; '.join(l))
return force_text(u'; '.join(l))
display.short_description = _('description')
@property
@ -161,7 +166,7 @@ class Validation(models.Model):
except UserDocument.DoesNotExist:
pass
def __unicode__(self):
def __str__(self):
return self.display()

View File

@ -1,4 +1,3 @@
import urllib
import logging
from json import dumps
from copy import deepcopy
@ -16,6 +15,7 @@ from django.contrib import messages
from django.contrib.auth import get_user_model, REDIRECT_FIELD_NAME
from django.contrib.auth import logout as auth_logout
from django.contrib.auth import views as auth_views
from django.utils.http import quote
from django.utils.translation import ugettext as _
from django.utils.decorators import method_decorator
from django.conf import settings
@ -262,7 +262,7 @@ def login(request, *args, **kwargs):
if not 'next' in request.GET:
return HttpResponseRedirect(resolve_url('mellon_login'))
return HttpResponseRedirect(resolve_url('mellon_login') + '?next='
+ urllib.quote(request.GET.get('next')))
+ quote(request.GET.get('next')))
return auth_views.login(request, *args, **kwargs)

View File

@ -14,13 +14,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import urlparse
import logging
import requests
from django.conf import settings
from django.utils.six.moves.urllib import parse as urlparse
from django.utils.translation import ugettext_lazy as _
from rest_framework.authentication import BasicAuthentication

View File

@ -20,7 +20,7 @@ class Command(BaseCommand):
def handle(self, redirect_uri, paths, client_id, **options):
client = OAuth2Client.objects.get(id=client_id)
for path in paths:
with open(path) as file_object:
with open(path, 'rb') as file_object:
filename = os.path.basename(path)
f = ContentFile(file_object.read(), name=filename)
document = Document.objects.get_by_file(f)

View File

@ -9,7 +9,7 @@ import fargo.oauth2.models
class Migration(migrations.Migration):
replaces = [(b'oauth2', '0001_initial'), (b'oauth2', '0002_auto_20180321_2343'), (b'oauth2', '0003_auto_20180322_1016'), (b'oauth2', '0004_auto_20180326_1330'), (b'oauth2', '0005_auto_20180331_1532')]
replaces = [('oauth2', '0001_initial'), ('oauth2', '0002_auto_20180321_2343'), ('oauth2', '0003_auto_20180322_1016'), ('oauth2', '0004_auto_20180326_1330'), ('oauth2', '0005_auto_20180331_1532')]
initial = True

View File

@ -21,6 +21,7 @@ from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.validators import URLValidator
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from django.db.models.query import QuerySet
from django.utils.timezone import now
@ -46,6 +47,7 @@ def validate_https_url(data):
raise ValidationError(errors)
@python_2_unicode_compatible
class OAuth2Client(models.Model):
client_name = models.CharField(max_length=255)
redirect_uris = models.TextField(
@ -63,7 +65,7 @@ class OAuth2Client(models.Model):
def check_redirect_uri(self, redirect_uri):
return redirect_uri in self.redirect_uris.strip().split()
def __unicode__(self):
def __str__(self):
return self.client_name
class Meta:

View File

@ -1,6 +1,7 @@
import cgi
from urllib import unquote
from django.utils import six
from django.utils.http import unquote
from django.utils.timezone import now
from django.conf import settings
@ -35,12 +36,13 @@ def get_content_disposition_value(request):
return None, 'wrong disposition type: attachment expected'
if 'filename*' in filename:
encode, country, name = filename['filename*'].split("'")
if six.PY3:
return (unquote(name, encode), None)
# check accepted charset from rfc 5987
if encode == 'UTF-8':
return unquote(name.decode('utf8')), None
return unquote(name).decode('utf8'), None
elif encode == 'ISO-8859-1':
return unquote(name.decode('iso-8859-1')), None
return unquote(name).decode('iso-8859-1'), None
else:
return None, 'unknown encoding: UTF-8 or ISO-8859-1 allowed'
elif 'filename' in filename:

View File

@ -16,9 +16,8 @@
import logging
from urllib import quote
from django.shortcuts import get_object_or_404
from django.utils.http import quote
from django.utils.translation import ugettext as _
from django.utils.timezone import now
from django.core.files.base import ContentFile
@ -154,7 +153,7 @@ def document_response(user_document):
content_type='application/octet-stream')
filename = user_document.filename
ascii_filename = filename.encode('ascii', 'replace')
ascii_filename = filename.encode('ascii', 'replace').decode()
percent_encoded_filename = quote(filename.encode('utf8'), safe='')
response['Content-Disposition'] = 'attachment; filename="%s"; filename*=UTF-8\'\'%s' % (ascii_filename,
percent_encoded_filename)

View File

@ -246,4 +246,4 @@ local_settings_file = os.environ.get('FARGO_SETTINGS_FILE',
'local_settings.py'))
if os.path.exists(local_settings_file):
execfile(local_settings_file)
exec(open(local_settings_file).read())

View File

@ -1,12 +1,12 @@
import urlparse
from django.utils.http import urlencode
from django.utils.six.moves.urllib import parse as urlparse
def make_url(__url, **kwargs):
request = kwargs.pop('request', None)
parsed = urlparse.urlparse(__url)
query = urlparse.parse_qs(parsed.query)
for key, value in kwargs.iteritems():
for key, value in kwargs.items():
if value is not None:
query[key] = value
parsed = parsed[:4] + (urlencode(query),) + parsed[5:]

View File

@ -1,3 +1,5 @@
# -*- coding: utf-8 -*-
import logging
import pytest
import django_webtest
@ -96,6 +98,4 @@ def document():
@pytest.fixture
def user_doc(document, john_doe):
return UserDocument.objects.create(user=john_doe, document=document, filename='Baudelaire.txt')
return UserDocument.objects.create(user=john_doe, document=document, filename='éléphant.txt')

View File

@ -65,7 +65,7 @@ def test_push_document(app, admin_user, john_doe):
== set(['origin', 'file_b64_content']))
data.update({
'origin': 'wcs',
'file_b64_content': base64.b64encode('coin'),
'file_b64_content': base64.b64encode(b'coin').decode(),
'file_name': 'monfichier.pdf',
})
response = app.post_json(url, params=data, status=200)
@ -77,13 +77,13 @@ def test_push_document(app, admin_user, john_doe):
assert models.UserDocument.objects.get().user == john_doe
assert models.UserDocument.objects.get().deletable_by_user == True
assert models.Document.objects.count() == 1
assert models.Document.objects.get().content.read() == 'coin'
assert models.Document.objects.get().content.read() == b'coin'
assert (models.UserDocument.objects.get().document
== models.Document.objects.get())
assert (models.UserDocument.objects.get().origin
== models.Origin.objects.get())
data['file_b64_content'] = base64.b64encode('coin2')
data['file_b64_content'] = base64.b64encode(b'coin2').decode()
data['deletable_by_user'] = False
response = app.post_json(url, params=data, status=200)
assert response.json['result'] == 1
@ -91,7 +91,7 @@ def test_push_document(app, admin_user, john_doe):
assert models.UserDocument.objects.count() == 2 # new document
assert models.UserDocument.objects.filter(deletable_by_user=False).count() == 1
data['file_b64_content'] = base64.b64encode('coin3')
data['file_b64_content'] = base64.b64encode(b'coin3').decode()
data['deletable_by_user'] = True
response = app.post_json(url, params=data, status=200)
assert response.json['result'] == 1
@ -99,7 +99,7 @@ def test_push_document(app, admin_user, john_doe):
assert models.UserDocument.objects.filter(deletable_by_user=True).count() == 2
# same document
data['file_b64_content'] = base64.b64encode('coin3')
data['file_b64_content'] = base64.b64encode(b'coin3').decode()
data['deletable_by_user'] = True
response = app.post_json(url, params=data, status=400)
assert response.json['result'] == 0
@ -114,14 +114,14 @@ def test_push_document_max_document_size(app, private_settings, admin_user, john
data = {
'user_email': john_doe.email,
'origin': 'wcs',
'file_b64_content': base64.b64encode('coin'),
'file_b64_content': base64.b64encode(b'coin').decode(),
'file_name': 'monfichier.pdf',
}
response = app.post_json(url, params=data, status=400)
assert response.json['result'] == 0
assert response.json['errors'].keys() == ['file_b64_content']
assert list(response.json['errors'].keys()) == ['file_b64_content']
assert response.json['errors']['file_b64_content'][0]['code'] == 'too-big'
assert response.json['errors']['file_b64_content'][0]['limit'] == 1
assert response.json['errors']['file_b64_content'][0]['limit'] == '1'
def test_push_document_max_document_box_size(app, private_settings, admin_user, john_doe):
@ -131,7 +131,7 @@ def test_push_document_max_document_box_size(app, private_settings, admin_user,
data = {
'user_email': john_doe.email,
'origin': 'wcs',
'file_b64_content': base64.b64encode('coin'),
'file_b64_content': base64.b64encode(b'coin').decode(),
'file_name': 'monfichier.pdf',
}
response = app.post_json(url, params=data, status=200)
@ -139,9 +139,9 @@ def test_push_document_max_document_box_size(app, private_settings, admin_user,
assert models.Document.objects.count() == 1
response = app.post_json(url, params=data, status=400)
assert response.json['result'] == 0
assert response.json['errors'].keys() == ['__all__']
assert list(response.json['errors'].keys()) == ['__all__']
assert response.json['errors']['__all__'][0]['code'] == 'box-is-full'
assert response.json['errors']['__all__'][0]['limit'] == 4
assert response.json['errors']['__all__'][0]['limit'] == '4'
def test_push_document_slashed_name(app, admin_user, john_doe):
@ -150,7 +150,7 @@ def test_push_document_slashed_name(app, admin_user, john_doe):
data = {
'user_email': john_doe.email,
'origin': 'wcs',
'file_b64_content': base64.b64encode('whatever'),
'file_b64_content': base64.b64encode(b'whatever').decode(),
'file_name': 'monfichier 18/06/2017.pdf',
}
response = app.post_json(url, params=data, status=200)

View File

@ -13,8 +13,8 @@ def test_cleanup(freezer, john_doe):
client = OAuth2Client.objects.create(client_name='c', redirect_uris='')
foo = Document.objects.create(content=ContentFile('foo', name='foo.txt'))
bar = Document.objects.create(content=ContentFile('bar', name='bar.txt'))
foo = Document.objects.create(content=ContentFile(b'foo', name='foo.txt'))
bar = Document.objects.create(content=ContentFile(b'bar', name='bar.txt'))
UserDocument.objects.create(user=john_doe,
document=foo,
filename='foo.txt',

View File

@ -21,7 +21,7 @@ def login(app, username='admin', password='admin', user=None):
return app
def test_document_delete(app):
f = ContentFile('A test file, ez pz.', 'test_file.txt')
f = ContentFile(b'A test file, ez pz.', 'test_file.txt')
doc = Document.objects.get_by_file(f)
file_path = doc.content.path
assert os.path.isfile(file_path)

View File

@ -1,13 +1,14 @@
# -*- coding: utf-8 -*-
import os
import json
import mock
import pytest
from urllib import quote
import urlparse
from django.core.urlresolvers import reverse
from django.utils.http import urlencode
from django.core.management import call_command
from django.utils.http import quote, urlencode
from django.utils.six.moves.urllib import parse as urlparse
from fargo.oauth2.models import OAuth2Client, OAuth2Authorize, OAuth2TempFile
from fargo.fargo.models import UserDocument
@ -44,7 +45,7 @@ def test_get_document_oauth2(app, john_doe, oauth2_client, user_doc):
}
# test missing redirect_uri
resp = app.get(url, params={}, status=400)
assert resp.content == 'missing redirect_uri parameter'
assert resp.text == 'missing redirect_uri parameter'
# test missing client id
params['redirect_uri'] = 'https://toto.example.com'
resp = app.get(url, params=params, status=302)
@ -65,7 +66,7 @@ def test_get_document_oauth2(app, john_doe, oauth2_client, user_doc):
assert resp.status_code == 200
assert len(resp.forms[0]['document'].options) == 2
options = resp.forms[0]['document'].options
assert 'Baudelaire.txt' in options[1]
assert u'éléphant.txt' in options[1]
resp.forms[0]['document'].select(options[1][0])
resp = resp.forms[0].submit()
@ -97,8 +98,8 @@ def test_get_document_oauth2(app, john_doe, oauth2_client, user_doc):
assert 'Content-disposition' in resp.headers
content_disposition = resp.content_disposition.replace(' ', '').split(';')
assert content_disposition[0] == 'attachment'
assert content_disposition[1] == 'filename="Baudelaire.txt"'
assert content_disposition[2] == 'filename*=UTF-8\'\'Baudelaire.txt'
assert content_disposition[1] == 'filename="?l?phant.txt"'
assert content_disposition[2] == 'filename*=UTF-8\'\'%C3%A9l%C3%A9phant.txt'
def test_put_document(app, john_doe, oauth2_client):
@ -111,10 +112,10 @@ def test_put_document(app, john_doe, oauth2_client):
app.authorization = ('Basic', (str(oauth2_client.client_id), str(oauth2_client.client_secret)))
resp = app.post(url, params=data, status=400)
assert 'missing content-disposition header' in resp.content
assert 'missing content-disposition header' in resp.text
filename = 'Baudelaire.txt'.encode('ascii', 'replace')
percent_encode_filename = quote(filename.encode('utf8'), safe='')
filename = 'éléphant.txt'
percent_encode_filename = quote(filename, safe='')
headers = {
'Content-disposition': 'attachment; filename="%s"; filename*=UTF-8\'\'%s' % (filename, percent_encode_filename)
}
@ -145,7 +146,7 @@ def test_put_document(app, john_doe, oauth2_client):
assert OAuth2TempFile.objects.count() == 1
assert UserDocument.objects.count() == 1
assert OAuth2TempFile.objects.get().document == UserDocument.objects.get().document
assert UserDocument.objects.filter(user=john_doe, document=doc.document, filename='Baudelaire.txt').exists()
assert UserDocument.objects.filter(user=john_doe, document=doc.document, filename=u'éléphant.txt').exists()
def test_confirm_put_document_file_exception(app, oauth2_client, john_doe, user_doc):
@ -158,12 +159,12 @@ def test_confirm_put_document_file_exception(app, oauth2_client, john_doe, user_
url = reverse('oauth2-put-document-authorize', kwargs={'pk': 'fakemofo'})
url += '?%s' % urlencode({'redirect_uri': 'https://example.com'})
resp = app.get(url)
assert 'The document has not been uploaded' in resp.content
assert 'The document has not been uploaded' in resp.text
url = reverse('oauth2-put-document-authorize', kwargs={'pk': oauth_tmp_file.pk})
url += '?%s' % urlencode({'redirect_uri': 'https://example.com'})
resp = app.get(url)
assert 'This document is already in your portfolio' in resp.content
assert 'This document is already in your portfolio' in resp.text
@mock.patch('fargo.oauth2.authentication.requests.post')
@ -180,7 +181,7 @@ def test_idp_authentication(mocked_post, settings, app, oauth2_client, john_doe,
params['redirect_uri'] = 'https://example.com'
resp = app.get(url, params=params)
options = resp.forms[0]['document'].options
assert 'Baudelaire.txt' in options[1]
assert u'éléphant.txt' in options[1]
resp.forms[0]['document'].select(options[1][0])
resp = resp.forms[0].submit()
auth = OAuth2Authorize.objects.filter(user_document__user=john_doe)[0]

View File

@ -2,9 +2,9 @@
from webtest import Upload
import pytest
import urlparse
from django.core.urlresolvers import reverse
from django.utils.six.moves.urllib import parse as urlparse
try:
import magic
@ -26,26 +26,26 @@ def test_upload(app, john_doe):
login(app, user=john_doe)
response1 = app.get('/')
form = response1.form
form['content'] = Upload('monfichier.pdf', 'coin', 'application/pdf')
form['content'] = Upload('monfichier.pdf', b'coin', 'application/pdf')
response2 = form.submit().follow()
assert 'monfichier.pdf' in response2.content
assert 'monfichier.pdf' in response2.text
if magic is not None:
assert UserDocument.objects.get(filename='monfichier.pdf').document.mime_type == 'text/plain'
assert ' mime-text ' in response2.content
assert ' mime-text-plain' in response2.content
assert ' mime-text ' in response2.text
assert ' mime-text-plain' in response2.text
UserDocument.objects.all().delete()
response1 = app.get('/')
form = response1.form
form['content'] = Upload('monfichier.pdf', '%PDF-1.4 ...', 'application/pdf')
form['content'] = Upload('monfichier.pdf', b'%PDF-1.4 ...', 'application/pdf')
response2 = form.submit().follow()
assert 'monfichier.pdf' in response2.content
assert '12 bytes' in response2.content
assert 'monfichier.pdf' in response2.text
assert u'12 bytes' in response2.text
if magic is not None:
assert UserDocument.objects.get(filename='monfichier.pdf').document.mime_type == 'application/pdf'
assert ' mime-application ' in response2.content
assert ' mime-application-pdf' in response2.content
assert ' mime-application ' in response2.text
assert ' mime-application-pdf' in response2.text
def test_upload_max_size(app, private_settings, john_doe):
@ -53,10 +53,10 @@ def test_upload_max_size(app, private_settings, john_doe):
login(app, user=john_doe)
response1 = app.get('/')
form = response1.form
form['content'] = Upload('monfichier.pdf', 'coin', 'application/pdf')
form['content'] = Upload('monfichier.pdf', b'coin', 'application/pdf')
response2 = form.submit()
assert response2.status_code == 200
assert 'Uploaded file is too big' in response2.content
assert 'Uploaded file is too big' in response2.text
def test_upload_max_document_box_size(app, private_settings, john_doe):
@ -64,15 +64,15 @@ def test_upload_max_document_box_size(app, private_settings, john_doe):
login(app, user=john_doe)
response1 = app.get('/')
form = response1.form
form['content'] = Upload('monfichier.pdf', 'coin', 'application/pdf')
form['content'] = Upload('monfichier.pdf', b'coin', 'application/pdf')
response2 = form.submit().follow()
assert 'monfichier.pdf' in response2.content
assert 'monfichier.pdf' in response2.text
response1 = app.get('/')
form = response1.forms['send-file']
form['content'] = Upload('monfichier.pdf', 'coin', 'application/pdf')
form['content'] = Upload('monfichier.pdf', b'coin', 'application/pdf')
response2 = form.submit()
assert response2.status_code == 200
assert 'Your document box is full (limit is 4)' in response2.content
assert 'Your document box is full (limit is 4)' in response2.text
def test_pick(app, private_settings, john_doe, user_doc):

View File

@ -1,6 +1,6 @@
[tox]
toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/fargo/
envlist = coverage-dj18-sqlite,coverage-dj111-sqlite,coverage-dj18-pg,coverage-dj111-pg
envlist = py2-coverage-dj18-sqlite,py2-coverage-dj111-sqlite,py2-coverage-dj18-pg,{py2,py3}-coverage-dj111-pg
[testenv]
usedevelop = True
@ -27,6 +27,7 @@ deps =
django-webtest<1.9.3
WebTest
djangorestframework>=3.3,<3.4
mock
commands =
py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests}
coverage: mv coverage.xml coverage-{envname}.xml