entrouvert
/
zoo
17
0
Fork 0
Code Issues Pull Requests Releases Activity

wip/74843-compatibilite-django #1

Merged
bdauvergne merged 11 commits from wip/74843-compatibilite-django into main 2023-02-27 10:38:51 +01:00
Conversation 0 Commits 11 Files Changed 33 +144 -241
33 changed files with 144 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
setup.py
View File

@ -106,13 +106,11 @@ setup(
'Programming Language :: Python',
],
install_requires=[
'django<2.3',
'django<3.3',
'isodate',
'psycopg2<2.9',
'jsonschema',
'gadjo',
'six',
'djangorestframework>=3.3,<3.10',
'djangorestframework>=3.9,<3.13',
'pytz',
'python-dateutil',
'django-admin-rangefilter',

4
tests/test_commands.py
View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import io
import mock
import pytest
import sys
@ -9,12 +10,11 @@ from zoo.zoo_nanterre import utils
from django.core.management import call_command
from django.db.transaction import atomic
from django.utils.six import StringIO
def get_output_of_command(command, *args, **kwargs):
old_stdout = sys.stdout
output = sys.stdout = StringIO()
output = sys.stdout = io.StringIO()
call_command(command, *args, **kwargs)
sys.stdout = old_stdout
return output.getvalue()

9
tests/test_nanterre.py
View File

@ -5,14 +5,13 @@ import datetime
import isodate
import requests
import threading
import urllib.parse
import pytest
import httmock
from django.urls import reverse
from django.utils.encoding import force_text
from django.utils.http import urlencode
from django.utils.six.moves.urllib import parse as urlparse
from django.utils.timezone import now
from zoo.zoo_data.models import Entity, Relation, Log, Job
@ -693,9 +692,9 @@ def test_create_individu(settings, transactional_db, app, app_noauth, rsu_schema
more = response.json.get('more')
if more:
assert 'cookie' in response.json
parsed = urlparse.urlparse(response.json['more'])
parsed = urllib.parse.urlparse(response.json['more'])
query = parsed.query
assert urlparse.parse_qs(query)['cookie'] == [response.json['cookie']]
assert urllib.parse.parse_qs(query)['cookie'] == [response.json['cookie']]
assert sorted(d['id'] for d in all_data) == sorted(qs.values_list('id', flat=True))
assert count == qs.count()
assert 'more' not in response.json
@ -1294,7 +1293,7 @@ def test_passage_a_la_majorite(db, settings, nanterre_classic_family, freezer):
Job.redo(timestamp=now() + datetime.timedelta(seconds=20))
assert len(requests) == 1
req_content = json.loads(force_text(requests[0].body))
req_content = json.loads(requests[0].body)
assert req_content['metadonnees']['service'] == 'passage-majorite'
assert len(req_content['fragments']) == 3
assert req_content['fragments'][0]['type'] == 'maj-adresse'

6
tests/test_nanterre_doublons.py
View File

@ -1,8 +1,8 @@
import copy
import urllib
from django.urls import reverse
from django.core.management import call_command
from django.utils.six.moves.urllib import parse as urlparse
from zoo.zoo_nanterre.models import Duplicate
from zoo.zoo_data.models import Log, Entity
@ -43,8 +43,8 @@ def test_list_doublons(nanterre_classic_family, app):
assert response.json['err'] == 0
assert 'more' in response.json
assert 'cookie' in response.json
assert response.json['cookie'] == urlparse.parse_qs(
urlparse.urlparse(
assert response.json['cookie'] == urllib.parse.parse_qs(
urllib.parse.urlparse(
response.json['more']).query)['cookie'][0]
assert len(response.json['data']) >= 10
assert response.json['data'][0]['id'] == d.id

7
tests/test_nanterre_fragments.py
View File

@ -4,7 +4,6 @@ import json
import httmock
from django.urls import reverse
from django.utils.encoding import force_text
from zoo.zoo_nanterre.fragments import Synchronization
from zoo.models import Job
@ -29,7 +28,7 @@ def test_synchro_full(app, nanterre_classic_family):
@httmock.urlmatch()
def technocarte_ok(url, request):
request_bodies.append(json.loads(force_text(request.body)))
request_bodies.append(json.loads(request.body))
return httmock.response(
200, [
{
@ -228,7 +227,7 @@ def test_infor(app, nanterre_classic_family):
@httmock.urlmatch()
def infor_ok(url, request):
request_bodies.append(json.loads(force_text(request.body)))
request_bodies.append(json.loads(request.body))
return httmock.response(
200, {
'http_code': 200,
@ -277,7 +276,7 @@ def test_infor(app, nanterre_classic_family):
@httmock.urlmatch()
def infor_nok(url, request):
request_bodies.append(json.loads(force_text(request.body)))
request_bodies.append(json.loads(request.body))
return httmock.response(
200, {
'http_code': 500,

15
tests/test_nanterre_synchronize_federations.py
View File

@ -1,10 +1,7 @@
# -*- coding: utf-8 -*-
import csv
from django.utils import six
from django.utils.encoding import force_bytes
from django.utils.six import StringIO
import io
from webtest import Upload
@ -26,19 +23,15 @@ def test_synchronize_federations(settings, app, nanterre_classic_family, admin):
response = response.click(u'Synchroniser les fédérations')
response = response.click(u'Nouvel import')
response.form.set('app_id', 'technocarte')
content = force_bytes('\n'.join(map(str, [f['kevin'].id + 1000, f['marie'].id + 1000, '99999'])))
content = ('\n'.join(map(str, [f['kevin'].id + 1000, f['marie'].id + 1000, '99999']))).encode()
response.form.set('csv_uploaded', Upload('federations.csv', content, 'application/octet-stream'))
response = response.form.submit().follow()
assert len(response.pyquery('table#result-list tbody tr')) == 1
response = response.click('Rapport')
def check_csv_response(csv_response):
if six.PY3:
reader = csv.DictReader(StringIO(csv_response.text))
reader.fieldnames = reader.reader.__next__()
else:
reader = csv.DictReader(StringIO(csv_response.content))
reader.fieldnames = reader.reader.next()
reader = csv.DictReader(io.StringIO(csv_response.text))
reader.fieldnames = reader.reader.__next__()
rows = list(reader)
def rows_by_action(action):

5
tox.ini
View File

@ -18,11 +18,12 @@ setenv =
usedevelop = true
deps =
dj22: django<2.3
dj22: psycopg2-binary<2.9
dj22: djangorestframework>=3.9.2,<3.10
dj32: django>=3.2,<3.3
djangorestframework>=3.9.2,<3.10
dj32: psycopg2-binary
pip>8
pytest-flakes
pg: psycopg2<2.9
coverage
pytest-cov
pytest-django

27
zoo/urls.py
View File

@ -14,30 +14,15 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""zoo URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.conf.urls import url, include
2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.urls import re_path, include
from django.contrib import admin
from .views import login, logout
urlpatterns = [
url(r'^admin/', admin.site.urls),
url(r'^demo/', include('zoo.zoo_demo.urls')),
url(r'^rsu/', include('zoo.zoo_nanterre.urls')),
url(r'^logout/$', logout, name='logout'),
url(r'^login/$', login, name='auth_login'),
re_path(r'^admin/', admin.site.urls),
re_path(r'^demo/', include('zoo.zoo_demo.urls')),
re_path(r'^rsu/', include('zoo.zoo_nanterre.urls')),
re_path(r'^logout/$', logout, name='logout'),
re_path(r'^login/$', login, name='auth_login'),
]

6
zoo/utils.py
View File

@ -16,8 +16,6 @@
import unicodedata
from django.utils.encoding import force_text
from rest_framework.views import exception_handler
from rest_framework.response import Response
@ -33,8 +31,8 @@ def rest_exception_handler(exc, context):
raise
response = Response({
'err': 1,
'exc_class': force_text(exc.__class__),
'exc_value': force_text(exc),
'exc_class': str(exc.__class__),
'exc_value': str(exc),
})
response.status = 400
return response

5
zoo/zoo_data/__init__.py
View File

@ -14,4 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
default_app_config = 'zoo.zoo_data.apps.ZooDataConfig'
import django
if django.VERSION < (3, 2):
default_app_config = 'zoo.zoo_data.apps.ZooDataConfig'

2
zoo/zoo_data/apps.py
View File

@ -15,7 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import gettext_lazy as _
class ZooDataConfig(AppConfig):

69
zoo/zoo_data/lookups.py
View File

@ -16,87 +16,40 @@
import django
from django.db.models import Transform, TextField, DateField
from django.db.models import Transform, CharField, DateField
from django.db.models.functions import Lower
from django.contrib.postgres.fields import jsonb
try:
from django.contrib.postgres.fields.jsonb import KeyTransform, KeyTransformTextLookupMixin
from django.db.models.fields.json import KeyTransform, KeyTransformTextLookupMixin
except ImportError:
# backport from Django 2.x
class KeyTransform(Transform):
operator = '->'
nested_operator = '#>'
def __init__(self, key_name, *args, **kwargs):
super(KeyTransform, self).__init__(*args, **kwargs)
self.key_name = key_name
def as_sql(self, compiler, connection):
key_transforms = [self.key_name]
previous = self.lhs
while isinstance(previous, KeyTransform):
key_transforms.insert(0, previous.key_name)
previous = previous.lhs
lhs, params = compiler.compile(previous)
if len(key_transforms) > 1:
return "(%s %s %%s)" % (lhs, self.nested_operator), [key_transforms] + params
try:
int(self.key_name)
except ValueError:
lookup = "'%s'" % self.key_name
else:
lookup = "%s" % self.key_name
return "(%s %s %s)" % (lhs, self.operator, lookup), params
jsonb.KeyTransform = KeyTransform
class KeyTextTransform(KeyTransform):
operator = '->>'
nested_operator = '#>>'
_output_field = TextField()
class KeyTransformTextLookupMixin(object):
"""
Mixin for combining with a lookup expecting a text lhs from a JSONField
key lookup. Make use of the ->> operator instead of casting key values to
text and performing the lookup on the resulting representation.
"""
def __init__(self, key_transform, *args, **kwargs):
assert isinstance(key_transform, KeyTransform)
key_text_transform = KeyTextTransform(
key_transform.key_name, *key_transform.source_expressions, **key_transform.extra
)
super(KeyTransformTextLookupMixin, self).__init__(key_text_transform, *args, **kwargs)
class Lower(Transform):
lookup_name = 'lower'
function = 'LOWER'
TextField.register_lookup(Lower)
from django.contrib.postgres.fields.jsonb import KeyTransform, KeyTransformTextLookupMixin
class Unaccent(Transform):
lookup_name = 'unaccent'
function = 'immutable_unaccent'
output_field = CharField()
TextField.register_lookup(Unaccent)
CharField.register_lookup(Unaccent)
CharField.register_lookup(Lower)
class Normalize(Transform):
lookup_name = 'normalize'
function = 'immutable_normalize'
output_field = CharField()
TextField.register_lookup(Normalize)
CharField.register_lookup(Normalize)
class Date(Transform):
lookup_name = 'timestamp'
function = 'immutable_date'
_output_field = DateField()
output_field = DateField()
TextField.register_lookup(Date)
CharField.register_lookup(Date)
class JSONUnaccent(KeyTransformTextLookupMixin, Unaccent):

20
zoo/zoo_data/models.py
View File

@ -24,22 +24,20 @@ import datetime
from django.db import models, connection
from django.db.models import F, Value
from django.db.models.functions import Lower
from django.db.models.query import QuerySet, Q
from django.core.exceptions import ValidationError
from django.urls import reverse
from django.utils.encoding import force_text
from django.utils.six import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import gettext_lazy as _
from django.utils.timezone import now
from django.contrib.postgres.fields import JSONField
from django.contrib.postgres.search import TrigramDistance
from .search import Unaccent, Lower, JSONTextRef
from .search import Unaccent, JSONTextRef
from zoo.zoo_meta.validators import schema_validator
@python_2_unicode_compatible
class Transaction(models.Model):
created = models.DateTimeField(
auto_now_add=True,
@ -54,7 +52,7 @@ class Transaction(models.Model):
null=True)
def __str__(self):
return force_text(self.id)
return str(self.id)
@classmethod
def get_transaction(self):
@ -77,7 +75,7 @@ class EntityQuerySet(QuerySet):
for key, value in kwargs.items():
filters.append(Q(**{
'content__' + key + '__unaccent__lower__trigram_similar':
Lower(Unaccent(Value(value))),
Unaccent(Lower(Value(value))),
}))
qs = qs.filter(functools.reduce(__or__, filters))
expressions = []
@ -93,7 +91,6 @@ class EntityQuerySet(QuerySet):
return qs
@python_2_unicode_compatible
class CommonData(models.Model):
def clean(self):
if self.schema:
@ -103,7 +100,7 @@ class CommonData(models.Model):
raise ValidationError({'content': e})
def __str__(self):
return force_text(self.id)
return str(self.id)
class Meta:
abstract = True
@ -255,7 +252,6 @@ class JobQuerySet(QuerySet):
return self.filter(**{'content__$classpath': class_path})
@python_2_unicode_compatible
class Job(models.Model):
'''Store synchronization messages sent to applications'''
SCHEDULER_STEP = 60 * 5 # 5 minutes
@ -334,7 +330,7 @@ class Job(models.Model):
job.state = cls.STATE_UNRECOVERABLE_ERROR
error = job.content.setdefault('error', {})
error['code'] = 'internal-server-error'
error['exc_detail'] = force_text(e)
error['exc_detail'] = str(e)
error['exc_tb'] = traceback.format_exc()
job.get_logger().exception('exception during job %s', job.admin_url)
job.save()
@ -366,7 +362,7 @@ class Job(models.Model):
url = self.admin_url
self.get_logger().exception('exception during job %s', url)
self.state = self.STATE_UNRECOVERABLE_ERROR
self.content['$exc_detail'] = force_text(e)
self.content['$exc_detail'] = str(e)
self.content['$exc_tb'] = traceback.format_exc()
self.content['$classpath'] = self.get_classpath(action)
self.save()

8
zoo/zoo_data/search.py
View File

@ -14,7 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.db.models import Func, Value
from django.db.models import Func, Value, CharField
class Unaccent(Func):
@ -27,11 +27,6 @@ class Normalize(Func):
arity = 1
class Lower(Func):
function = 'LOWER'
arity = 1
class JSONRef(Func):
function = ''
arg_joiner = '->'
@ -46,6 +41,7 @@ class JSONTextRef(Func):
function = ''
arg_joiner = '->>'
arity = 2
output_field = CharField()
def __init__(self, *expressions, **extra):
jsonb = expressions[0]

2
zoo/zoo_data/static/js/jsoneditor.min.js vendored
View File

@ -856,4 +856,4 @@ c&&c.destroy(),
c=new f(this.get(0),a),this.data("jsoneditor",c),
// Setup event listeners
c.on("change",function(){b.trigger("change")}),c.on("ready",function(){b.trigger("ready")}))}return this}}}(),window.JSONEditor=f}();
//# sourceMappingURL=jsoneditor.min.js.map
//# sourceMappingURL=jsoneditor.min.js.map

6
zoo/zoo_demo/urls.py
View File

@ -14,11 +14,11 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.conf.urls import url
from django.urls import re_path
from .views import schemas, schema
urlpatterns = [
url(r'^schemas/$', schemas, name='demo-schemas'),
url(r'^schemas/(?P<name>\w*)/$', schema, name='demo-schema'),
re_path(r'^schemas/$', schemas, name='demo-schemas'),
re_path(r'^schemas/(?P<name>\w*)/$', schema, name='demo-schema'),
]

5
zoo/zoo_meta/__init__.py
View File

@ -14,4 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
default_app_config = 'zoo.zoo_meta.apps.ZooMetaAppConfig'
import django
if django.VERSION < (3, 2):
default_app_config = 'zoo.zoo_meta.apps.ZooMetaAppConfig'

2
zoo/zoo_meta/apps.py
View File

@ -14,7 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import gettext_lazy as _
from django.db.models.signals import post_migrate, post_save
from django.apps import AppConfig

3
zoo/zoo_meta/management/commands/zoo-rebuild-indexes.py
View File

@ -16,7 +16,6 @@
from __future__ import print_function
from django.core.management.base import BaseCommand
from django.utils.encoding import force_text
from zoo.zoo_meta.models import EntitySchema
@ -25,7 +24,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
for schema in EntitySchema.objects.all():
if options['verbosity'] >= 1:
print('Rebuilding index for', force_text(schema), end=' ')
print('Rebuilding index for', schema, end=' ')
schema.rebuild_indexes()
if options['verbosity'] >= 1:
print(' Done.')

13
zoo/zoo_meta/models.py
View File

@ -18,8 +18,7 @@ from hashlib import md5
from django.apps import apps
from django.db import models, connection
from django.utils.encoding import force_bytes, force_text
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import gettext_lazy as _
from django.contrib.postgres.fields import JSONField
@ -74,7 +73,7 @@ class CommonSchema(models.Model):
def rebuild_string_index(self, cursor, table, path):
expr = 'immutable_normalize((content%s))' % self.path_to_sql_expr(path)
key = md5(force_bytes(expr)).hexdigest()[:8]
key = md5(expr.encode()).hexdigest()[:8]
sql = ('CREATE INDEX zoo_entity_%s_gin_%s_dynamic_idx ON %s USING gin ((%s) '
' gin_trgm_ops) WHERE schema_id = %s' % (key, self.id, table, expr, self.id))
cursor.execute(sql)
@ -84,7 +83,7 @@ class CommonSchema(models.Model):
def rebuild_string_date_time_index(self, cursor, table, path):
expr = 'immutable_date(content%s)' % self.path_to_sql_expr(path)
key = md5(force_bytes(expr)).hexdigest()[:8]
key = md5(expr.encode()).hexdigest()[:8]
sql = ('CREATE INDEX zoo_entity_%s_%s_dynamic_idx ON %s (%s) '
'WHERE schema_id = %s' % (key, self.id, table, expr, self.id))
cursor.execute(sql)
@ -99,7 +98,7 @@ class CommonSchema(models.Model):
else:
raise NotImplementedError(self)
key = md5(force_bytes(expr)).hexdigest()[:8]
key = md5(expr.encode()).hexdigest()[:8]
gin_sql = ('CREATE INDEX zoo_entity_%s_gin_%s_dynamic_idx ON %s USING gin ((%s) '
'gin_trgm_ops) WHERE schema_id = %s' % (key, self.id, table, expr, self.id))
gist_sql = ('CREATE INDEX zoo_entity_%s_gist_%s_dynamic_idx ON %s USING gist ((%s)'
@ -147,8 +146,8 @@ class CommonSchema(models.Model):
try:
return eval(self.caption_template, {}, value.content)
except Exception as e:
return force_text(e)
return force_text(value.id)
return str(e)
return str(value.id)
class Meta:
abstract = True

5
zoo/zoo_nanterre/__init__.py
View File

@ -14,4 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
default_app_config = 'zoo.zoo_nanterre.apps.ZooNanterreConfig'
import django
if django.VERSION < (3, 2):
default_app_config = 'zoo.zoo_nanterre.apps.ZooNanterreConfig'

7
zoo/zoo_nanterre/api_views.py
View File

@ -32,7 +32,6 @@ from django.db.models.query import Q
from django.db.transaction import non_atomic_requests, atomic
from django.urls import reverse
from django.http import Http404, HttpResponse
from django.utils.encoding import force_text
from django.utils.timezone import now
from django.utils.http import urlencode
@ -135,7 +134,7 @@ class TransactionalView(APIView):
content = {
'request': self.request.data,
'status_code': 500,
'$exc_detail': force_text(exc),
'$exc_detail': str(exc),
'$exc_tb': traceback.format_exc(),
}
self.transaction.content = content
@ -2055,7 +2054,7 @@ class FalsePositiveView(DoublonActionView):
except AssertionError as e:
return Response({
'err': 1,
'errors': force_text(e),
'errors': str(e),
}, status=500)
@ -2096,7 +2095,7 @@ class DedupView(DoublonActionView):
except AssertionError as e:
return Response({
'err': 1,
'errors': force_text(e),
'errors': str(e),
}, status=500)
dedup = DedupView.as_view()

8
zoo/zoo_nanterre/apps.py
View File

@ -19,8 +19,8 @@
import functools
from django.apps import AppConfig
from django.conf.urls import url
from django.utils.translation import ugettext_lazy as _
from django.urls import re_path
from django.utils.translation import gettext_lazy as _
from django.db.models.signals import post_migrate
@ -93,14 +93,14 @@ class ZooNanterreConfig(AppConfig):
urls = []
for desc in descs:
urls.append(url(
urls.append(re_path(
r'^synchronize-federations/%s$' % desc['re'],
model_admin.admin_site.admin_view(
getattr(views, 'synchronize_federations' + desc['view'])),
kwargs={'model_admin': model_admin},
name='synchronize-federations' + desc['name'],
))
urls.append(url(
urls.append(re_path(
r'^inactive/',
model_admin.admin_site.admin_view(
getattr(views, 'inactive_index')),

11
zoo/zoo_nanterre/forms.py
View File

@ -15,8 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django import forms
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import gettext_lazy as _
from django.core.exceptions import ValidationError
from .utils import PersonSearch, get_applications, get_application
@ -63,13 +62,15 @@ class SynchronizeFederationsForm(forms.Form):
def clean_csv_uploaded(self):
csv_uploaded = self.cleaned_data['csv_uploaded']
errors = []
csv_uploaded.seek(0)
for i, line in enumerate(csv_uploaded):
try:
force_text(line).encode('ascii')
# works with pyhton2 and 3
except (UnicodeEncodeError, UnicodeDecodeError) as e:
line.decode('ascii')
except UnicodeError as e:
errors.append(_(u'non-ASCII character on line {0} and column {1}').format(
i + 1, e.start + 1))
# restore file state
csv_uploaded.seek(0)
if errors:
raise ValidationError(errors)
return csv_uploaded

3
zoo/zoo_nanterre/fragments.py
View File

@ -5,7 +5,6 @@ import datetime
import requests
from requests.exceptions import RequestException
from django.utils.encoding import force_text
from django.utils.timezone import now
from django.conf import settings
from django.db import DatabaseError
@ -204,7 +203,7 @@ class FragmentBuilder(object):
error_detail = u'erreur réseau/SSL ou expiration'
self.error = {
'code': 'transport-error',
'detail': force_text(e),
'detail': str(e),
}
state = self.state_on_network_error
else:

2
zoo/zoo_nanterre/management/commands/rsu-duplicates.py
View File

@ -23,7 +23,6 @@ import datetime
import django
from django.core.management.base import BaseCommand, CommandParser
from django.utils.six import python_2_unicode_compatible
from django.utils.timezone import now
from zoo.zoo_nanterre.utils import individu_caption
@ -31,7 +30,6 @@ from zoo.zoo_nanterre.duplicates import find_duplicates
from zoo.zoo_nanterre.models import Duplicate
@python_2_unicode_compatible
class Table(object):
def __init__(self, names):
self.size = len(names)

2
zoo/zoo_nanterre/models.py
View File

@ -14,7 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import gettext_lazy as _
from django.contrib.postgres.fields import JSONField
from django.db import models

9
zoo/zoo_nanterre/qf.py
View File

@ -8,7 +8,6 @@ from zoo.models import Job
import requests
from django.utils.encoding import force_text
from django.utils.timezone import now
from django.conf import settings
@ -64,7 +63,7 @@ class QF(object):
if response:
for qf in response:
try:
qf['annee_imposition'] = force_text(int(re.findall(r'(\d+)', qf['libelle'])[0]) - 1)
qf['annee_imposition'] = str(int(re.findall(r'(\d+)', qf['libelle'])[0]) - 1)
except Exception:
qf['annee_imposition'] = 'inconnue'
return response, error
@ -148,7 +147,7 @@ class QF(object):
else:
return response[0], None
else:
return None, u'Implicit calcul-qf réponse invalide: %r' % force_text(response)[:1024]
return None, 'Implicit calcul-qf réponse invalide: %r' % str(response)[:1024]
def lire_quotient_familial(self, individu, date_de_reference):
federation = individu.content['cles_de_federation'].get('implicit')
@ -168,8 +167,8 @@ class QF(object):
if isinstance(response, list):
return response, None
else:
return None, (u'Implicit lire-quotient-familial réponse invalide: %r'
% force_text(response)[:1024])
return None, ('Implicit lire-quotient-familial réponse invalide: %r'
% str(response)[:1024])
def editer_carte(self, individu, id_qf):
federation = individu.content['cles_de_federation'].get('implicit')

10
zoo/zoo_nanterre/saga.py
View File

@ -1,11 +1,11 @@
# -*- coding: utf-8 -*-
import datetime
import decimal
import urllib.parse
import xml.etree.ElementTree as ET
from django.conf import settings
from django.utils import six, timezone
from django.utils.six.moves.urllib import parse as urlparse
from django.utils import timezone
import requests
@ -36,13 +36,13 @@ class Saga(object):
@property
def creance_url(self):
return urlparse.urljoin(
return urllib.parse.urljoin(
self.url,
'/%s/services/etat_facture_creance_literal' % self.base_uri)
@property
def paiement_url(self):
return urlparse.urljoin(
return urllib.parse.urljoin(
self.url,
'/%s/services/paiement_internet_ws_literal' % self.base_uri)
@ -194,7 +194,7 @@ class Saga(object):
<urlretour_synchrone>{urlretour_synchrone}</urlretour_synchrone>
</Transaction>'''
assert factures, u'factures ne doit pas être vide'
id_facture = u'--'.join(six.text_type(facture.num) for facture in factures)
id_facture = u'--'.join(str(facture.num) for facture in factures)
montant = sum(facture.reste_a_payer for facture in factures)
tree, error = self.soap_call(
self.paiement_url, body, 'TransactionReturn',

11
zoo/zoo_nanterre/synchronize_federations.py
View File

@ -17,15 +17,13 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import csv
import io
from django.core.files.storage import default_storage
from django.urls import reverse
from django.conf import settings
from django.db import DatabaseError
from django.db.transaction import atomic
from django.utils import six
from django.utils.encoding import force_bytes
from django.utils.six import StringIO
from zoo.zoo_meta.models import EntitySchema
from zoo.zoo_data.models import Job, Entity, Transaction, Log
@ -106,15 +104,12 @@ class SynchronizeFederationsImport(object):
self.report('report')
def report(self, target):
output_file = StringIO()
output_file = io.StringIO()
writer = csv.writer(output_file)
writer.writerow(['RSU ID', 'prenoms', 'nom de naissance',
'nom d\'usage', 'application', 'federation', 'action'])
for action in self.actions:
if six.PY3:
action = [v for v in action]
else:
action = [force_bytes(v) for v in action]
action = [v for v in action]
writer.writerow(action)
setattr(self.action, target + '_csv_filename',
self.action.csv_filename + '-report.csv')

64
zoo/zoo_nanterre/urls.py
View File

@ -14,74 +14,74 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.conf.urls import url
from django.urls import re_path
from .views import demo, search, import_control
from . import api_views
urlpatterns = [
url(r'^demo/$', demo, name='demo'),
url(r'^demo/search/$', search, name='demo'),
url(r'^import_control/$', import_control, name='demo'),
url(r'^search/$', api_views.search, name='rsu-api-search'),
url(r'^individu/(?P<identifier>[-\w]+)/$', api_views.reseau, name='rsu-api-reseau'),
url(r'^individu/(?P<identifier>[-\w]+)/suppression/$', api_views.suppression_individu,
re_path(r'^demo/$', demo, name='demo'),
re_path(r'^demo/search/$', search, name='demo'),
re_path(r'^import_control/$', import_control, name='demo'),
re_path(r'^search/$', api_views.search, name='rsu-api-search'),
re_path(r'^individu/(?P<identifier>[-\w]+)/$', api_views.reseau, name='rsu-api-reseau'),
re_path(r'^individu/(?P<identifier>[-\w]+)/suppression/$', api_views.suppression_individu,
name='rsu-api-suppression-individu'),
url(r'^individu/(?P<identifier>[-\w]+)/liste/$', api_views.reseau_liste,
re_path(r'^individu/(?P<identifier>[-\w]+)/liste/$', api_views.reseau_liste,
name='rsu-api-reseau-liste'),
url(r'^individu/(?P<identifier>[-\w]+)/journal/$', api_views.journal, name='rsu-api-journal'),
url(r'^individu/(?P<identifier>[-\w]+)/declaration-responsabilite-legale/$',
re_path(r'^individu/(?P<identifier>[-\w]+)/journal/$', api_views.journal, name='rsu-api-journal'),
re_path(r'^individu/(?P<identifier>[-\w]+)/declaration-responsabilite-legale/$',
api_views.declaration_responsabilite_legale,
name='rsu-api-declaration-responsabilite-legale'),
url(r'^individu/(?P<identifier>[-\w]+)/declaration-adresse-principale/$',
re_path(r'^individu/(?P<identifier>[-\w]+)/declaration-adresse-principale/$',
api_views.declaration_adresse_principale,
name='rsu-api-declaration-adresse-principale'),
url(r'^individu/(?P<identifier>[-\w]+)/changement-de-situation-maritale/$',
re_path(r'^individu/(?P<identifier>[-\w]+)/changement-de-situation-maritale/$',
api_views.changement_de_situation_maritale,
name='rsu-api-changement-de-situation-maritale'),
url(r'^individu/(?P<identifier>[-\w]+)/separation/$',
re_path(r'^individu/(?P<identifier>[-\w]+)/separation/$',
api_views.separation, name='rsu-api-separation'),
url(r'^individu/(?P<identifier>[-\w]+)/declaration-de-deces/$',
re_path(r'^individu/(?P<identifier>[-\w]+)/declaration-de-deces/$',
api_views.declaration_de_deces, name='rsu-api-declaration-de-deces'),
url(r'^individu/(?P<identifier>[-\w]+)/(?P<identifier_enfant>[-\w]+)/'
re_path(r'^individu/(?P<identifier>[-\w]+)/(?P<identifier_enfant>[-\w]+)/'
'suppression-lien-de-responsabilite/$',
api_views.suppression_lien_de_responsabilite,
name='rsu-api-suppression-lien-de-responsabilite'),
url(r'^individu/$', api_views.create_individu, name='rsu-api-create-individu'),
url(r'^individu/(?P<identifier>[-\w]+)/federation/(?P<application>\w+)/$', api_views.federation,
re_path(r'^individu/$', api_views.create_individu, name='rsu-api-create-individu'),
re_path(r'^individu/(?P<identifier>[-\w]+)/federation/(?P<application>\w+)/$', api_views.federation,
name='rsu-api-federation'),
url(r'^declaration-union/$', api_views.declaration_union,
re_path(r'^declaration-union/$', api_views.declaration_union,
name='rsu-api-declaration-union'),
url(r'^synchronisation/$', api_views.synchronization,
re_path(r'^synchronisation/$', api_views.synchronization,
name='rsu-api-synchronization'),
url(r'^saga/retour-asynchrone/$', api_views.saga_retour_asynchrone,
re_path(r'^saga/retour-asynchrone/$', api_views.saga_retour_asynchrone,
name='rsu-api-saga-retour-asynchrone'),
url(r'^saga/retour-synchrone/$', api_views.saga_retour_synchrone,
re_path(r'^saga/retour-synchrone/$', api_views.saga_retour_synchrone,
name='rsu-api-saga-retour-synchrone'),
url(r'^saga/tiers/(?P<application>\w+)/(?P<identifier>[-\w]+)/$', api_views.saga_tiers,
re_path(r'^saga/tiers/(?P<application>\w+)/(?P<identifier>[-\w]+)/$', api_views.saga_tiers,
name='rsu-api-saga-tiers'),
url(r'^saga/(?P<identifier>[-\w]+)/factures/$', api_views.saga_factures,
re_path(r'^saga/(?P<identifier>[-\w]+)/factures/$', api_views.saga_factures,
name='rsu-api-saga-factures'),
url(r'^saga/(?P<identifier>[-\w]+)/transaction/$', api_views.saga_transaction,
re_path(r'^saga/(?P<identifier>[-\w]+)/transaction/$', api_views.saga_transaction,
name='rsu-api-saga-transaction'),
url(r'^qf/lire-quotients-valides/$', api_views.qf_lire_quotiens_valides,
re_path(r'^qf/lire-quotients-valides/$', api_views.qf_lire_quotiens_valides,
name='rsu-api-qf-lire-quotients-valides'),
url(r'^qf/simuler/$', api_views.qf_simuler,
re_path(r'^qf/simuler/$', api_views.qf_simuler,
name='rsu-api-qf-simuler'),
url(r'^qf/(?P<identifier>[-\w]+)/$', api_views.qf_calculer,
re_path(r'^qf/(?P<identifier>[-\w]+)/$', api_views.qf_calculer,
name='rsu-api-qf-calculer'),
url(r'^qf/(?P<identifier>[-\w]+)/editer-carte/(?P<id_qf>\w+)/$', api_views.qf_editer_carte,
re_path(r'^qf/(?P<identifier>[-\w]+)/editer-carte/(?P<id_qf>\w+)/$', api_views.qf_editer_carte,
name='rsu-api-qf-editer-carte'),
url(r'^doublons/$', api_views.doublons,
re_path(r'^doublons/$', api_views.doublons,
name='rsu-api-doublons'),
url(r'^doublons/(?P<doublon_id>[0-9 ]+)/$', api_views.doublon,
re_path(r'^doublons/(?P<doublon_id>[0-9 ]+)/$', api_views.doublon,
name='rsu-api-doublon'),
url(r'^doublons/(?P<doublon_id>[0-9 ]+)/false-positive/$', api_views.false_positive,
re_path(r'^doublons/(?P<doublon_id>[0-9 ]+)/false-positive/$', api_views.false_positive,
name='rsu-api-doublon-false-positive'),
url(r'^doublons/(?P<doublon_id>[0-9 ]+)/dedup/$', api_views.dedup,
re_path(r'^doublons/(?P<doublon_id>[0-9 ]+)/dedup/$', api_views.dedup,
name='rsu-api-doublon-dedup'),
]

24
zoo/zoo_nanterre/utils.py
View File

@ -38,13 +38,12 @@ import psycopg2
from django.conf import settings
from django.contrib.postgres.search import TrigramDistance
from django.db import connection
from django.db.models import Q, F, Value, ExpressionWrapper, CharField, When, Case
from django.db.models import Q, F, Value, ExpressionWrapper, CharField, When, Case, CharField
from django.db.models.functions import Least, Greatest, Coalesce, Concat
from django.db import transaction
from django.contrib.auth.hashers import make_password
from django.http import HttpResponse
from django.utils import six
from django.utils.timezone import now, make_aware
from django.utils.encoding import force_bytes
@ -542,7 +541,7 @@ class PersonSearch(object):
Coalesce(
JSONTextRef(F('content'), 'nom_d_usage'),
JSONTextRef(F('content'), 'nom_de_naissance'),
Value(' ')
Value(' '),
),
Value(' '),
JSONTextRef(F('content'), 'prenoms'))
@ -653,7 +652,7 @@ def integrity_check():
def upper_dict(d):
'''Transform all string values in d to uppercase'''
for key, value in d.items():
if isinstance(value, six.text_type):
if isinstance(value, str):
d[key] = value.upper()
@ -1283,18 +1282,11 @@ def individu_caption(individu):
def csv_export_response(rows, filename):
if six.PY3:
with io.StringIO(newline='') as f:
writer = csv.writer(f)
for row in rows:
writer.writerow(map(str, row))
r = HttpResponse(f.getvalue(), content_type='text/csv')
else:
with io.BytesIO() as f:
writer = csv.writer(f)
for row in rows:
writer.writerow(map(force_bytes, row))
r = HttpResponse(f.getvalue(), content_type='text/csv')
with io.StringIO(newline='') as f:
writer = csv.writer(f)
for row in rows:
writer.writerow(map(str, row))
r = HttpResponse(f.getvalue(), content_type='text/csv')
r['Content-Disposition'] = 'attachment; filename="%s"' % filename
return r

9
zoo/zoo_nanterre/views.py
View File

@ -30,7 +30,6 @@ from django.db import connection
from django.conf import settings
from django.core.cache import cache
from django.utils.timezone import now
from django.utils import six
from django.contrib.auth.decorators import permission_required
from django.contrib import messages
@ -117,9 +116,7 @@ def synchronize_federations_report(request, job_id, model_admin, *args, **kwargs
if not report:
raise Http404('no report')
with report:
text_report = report
if six.PY3:
text_report = io.TextIOWrapper(text_report, encoding='utf-8')
text_report = io.TextIOWrapper(report, encoding='utf-8')
reader = csv.reader(text_report)
next(reader)
actions = [row for row in reader if row[6] != 'KEEP']
@ -152,9 +149,7 @@ def synchronize_federations_apply_report(request, job_id, model_admin, *args, **
with report:
if not report:
raise Http404('no report')
text_report = report
if six.PY3:
text_report = io.TextIOWrapper(text_report, encoding='utf-8')
text_report = io.TextIOWrapper(report, encoding='utf-8')
reader = csv.reader(text_report)
next(reader)
actions = [row for row in reader if row[6] != 'KEEP']