diff --git a/debian/control b/debian/control
index 7a06f0c..d8461cf 100644
--- a/debian/control
+++ b/debian/control
@@ -7,6 +7,8 @@ Standards-Version: 3.9.1
Package: python-passerelle-reunion-fsn
Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}
+Depends: ${misc:Depends},
+ ${python:Depends},
+ python-unicodecsv
Description: Passerelle CR Reunion FSN
diff --git a/passerelle_reunion_fsn/migrations/0004_wcs_batches.py b/passerelle_reunion_fsn/migrations/0004_wcs_batches.py
new file mode 100644
index 0000000..cbe2574
--- /dev/null
+++ b/passerelle_reunion_fsn/migrations/0004_wcs_batches.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2020-04-24 15:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import jsonfield.fields
+import passerelle_reunion_fsn.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('passerelle_reunion_fsn', '0003_csv_checksum'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Batch',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('batch_date', models.DateField()),
+ ('last_update_datetime', models.DateTimeField(auto_now=True)),
+ ],
+ options={
+ 'ordering': ('batch_date',),
+ },
+ ),
+ migrations.CreateModel(
+ name='BatchFile',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('csv_file', models.FileField(upload_to=passerelle_reunion_fsn.models.batch_csv_file_location)),
+ ('csv_filename', models.CharField(max_length=256)),
+ ('ready', models.BooleanField(default=False)),
+ ('last_update_datetime', models.DateTimeField(auto_now=True)),
+ ('batch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='passerelle_reunion_fsn.Batch')),
+ ],
+ ),
+ migrations.AddField(
+ model_name='fsnreunionconnector',
+ name='wcs_form_slug',
+ field=models.CharField(blank=True, max_length=256, verbose_name='WCS form slug'),
+ ),
+ migrations.AddField(
+ model_name='fsnreunionconnector',
+ name='wcs_options',
+ field=jsonfield.fields.JSONField(blank=True, null=True),
+ ),
+ migrations.AddField(
+ model_name='batch',
+ name='resource',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='batches', to='passerelle_reunion_fsn.FSNReunionConnector'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='batch',
+ unique_together=set([('resource', 'batch_date')]),
+ ),
+ ]
diff --git a/passerelle_reunion_fsn/models.py b/passerelle_reunion_fsn/models.py
index e4dcaae..968618f 100644
--- a/passerelle_reunion_fsn/models.py
+++ b/passerelle_reunion_fsn/models.py
@@ -17,21 +17,52 @@
# along with this program. If not, see .
import base64
-import csv
+from datetime import datetime
import hashlib
from io import BytesIO
+import os
+import os.path
+import tempfile
+from django.conf import settings
+from django.core.files import File
from django.core.urlresolvers import reverse
from django.db import models, transaction
-from django.http import FileResponse
-from django.utils.dateparse import parse_datetime
-from django.utils.encoding import force_str, smart_text
+from django.http import FileResponse, HttpResponse
+from django.utils import dateformat, six
+from django.utils.dateparse import parse_date, parse_datetime
+from django.utils.encoding import force_str, force_text, smart_text
+from django.utils.six.moves.urllib import parse as urlparse
from django.utils.translation import ugettext_lazy as _
-
+import jsonfield
from passerelle.base.models import BaseResource
+from passerelle.base.signature import sign_url
from passerelle.utils.api import endpoint
from passerelle.utils.jsonresponse import APIError
+if six.PY3:
+ import csv
+else:
+ import unicodecsv as csv
+
+
+GENERATE_BATCH_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "title": "Generate batch",
+ "description": "",
+ "type": "object",
+ "properties": {
+ "batch_date": {
+ "description": "Bacth date",
+ "type": "string",
+ },
+ "force": {
+ "description": "Force",
+ "type": "boolean",
+ }
+ }
+}
+
COLUMNS_KEYNAMES = [
'code_application',
@@ -67,6 +98,14 @@ def csv_file_location(instance, filename):
return 'fsn_reunion/%s/%s' % (instance.ds_id, filename)
+def batch_csv_file_location(instance, filename):
+ return 'fsn_reunion/%s/batch/%s/%s/%s' % (
+ instance.batch.resource.id,
+ dateformat.format(instance.batch.batch_date, 'Y-m-d'),
+ instance.id, filename
+ )
+
+
class FSNReunionConnector(BaseResource):
category = _('Business Process Connectors')
@@ -81,6 +120,14 @@ class FSNReunionConnector(BaseResource):
max_length=256, blank=True, verbose_name=_('Instructeur identifier'),
help_text=_('Region identifier for this case')
)
+ wcs_form_slug = models.CharField(max_length=256, blank=True, verbose_name=_('WCS form slug'))
+ wcs_options = jsonfield.JSONField(null=True, blank=True)
+
+ # def daily(self):
+ # self.logger.info('start generate batch')
+ # for batch in self._generate_batches():
+ # self.logger.info('created batch %s' % batch.batch_date)
+ # self.logger.info('end generate batch')
def _ds_call(self, query, variables):
headers = {
@@ -105,6 +152,58 @@ class FSNReunionConnector(BaseResource):
return json['data']
+ def _wcs_call(self, filters={}):
+ if not getattr(settings, 'KNOWN_SERVICES', {}).get('wcs'):
+ raise APIError('No wcs found')
+
+ wcs_service = list(settings.KNOWN_SERVICES['wcs'].values())[0]
+ if self.wcs_options and 'instance' in self.wcs_options:
+ wcs_service = self.wcs_options['instance']
+ base_url = wcs_service['url']
+ orig = wcs_service.get('orig')
+ secret = wcs_service.get('secret')
+ limit = 10
+ params = {
+ 'orig': orig,
+ 'full': 'on',
+ 'limit': limit,
+ 'order_by': '-receipt_time'
+ }
+ params.update(self.wcs_options.get('filters', {}))
+ params.update(filters)
+
+ offset = 0
+ has_data = True
+ while has_data:
+ params['offset'] = offset
+ query_string = urlparse.urlencode(params)
+ api_url = sign_url(
+ urlparse.urljoin(
+ base_url,
+ 'api/forms/%s/list?%s' % (self.wcs_form_slug, query_string)
+ ),
+ key=secret
+ )
+ response = self.requests.get(api_url)
+ if response.status_code != 200:
+ raise APIError('Error fetching data from wcs')
+ data = response.json()
+ if not data:
+ has_data = False
+ else:
+ for form in data:
+ yield form
+ offset += limit
+
+ @endpoint(
+ methods=['get'], perm='can_access', name='wcs-call',
+ description=_('Call wcs debug')
+ )
+ def wcs_call(self, request):
+ return {
+ 'data': [form for form in self._wcs_call()]
+ }
+
@endpoint(
methods=['get'], perm='can_access', name='dsproxy-get-instructeurs',
description=_('DS Proxy: get instructeurs')
@@ -418,6 +517,216 @@ query getDossiers($demarcheNumber: Int!, $createdSince: ISO8601DateTime, $first:
'data': {}
}
+ @endpoint(
+ methods=['get'], perm='can_access', name='get-batches',
+ description=_('Get batches')
+ )
+ def get_batches(self, request):
+ res = []
+ for batch in self.batches.all():
+ res.append(
+ {
+ 'id': batch.pk,
+ 'text': dateformat.format(batch.batch_date, 'l d F Y'),
+ 'url': request.build_absolute_uri(batch.get_absolute_url())
+ }
+ )
+ return {'data': res}
+
+ @endpoint(
+ methods=['get'], perm='can_access', name='batchfile',
+ pattern='^(?P\w+)/$', example_pattern='{batchfile_pk}/',
+ parameters={
+ 'batchfile_pk': {
+ 'description': _('Batch file identifier'),
+ 'example_value': '2'
+ }
+ }, description=_('Get batch file')
+
+ )
+ def batchfile(self, request, batchfile_pk):
+ try:
+ batch_file = BatchFile.objects.get(pk=batchfile_pk)
+ except BatchFile.DoesNotExist:
+ raise APIError('Unkwon batch file identifier')
+ response = HttpResponse(
+ batch_file.csv_file.read(), content_type="text/csv"
+ )
+ response['Content-Disposition'] = 'attachment; filename="%s"' % batch_file.csv_filename
+ return response
+
+ @endpoint(
+ methods=['get'], perm='can_access', name='batch',
+ description=_('Get batch'),
+ pattern='^(?P\w+)/$', example_pattern='{batch_pk}/',
+ parameters={
+ 'batch_pk': {
+ 'description': _('Batch identifier'),
+ 'example_value': '2'
+ }
+ }
+ )
+ def batch(self, request, batch_pk):
+ try:
+ batch = self.batches.get(pk=batch_pk)
+ except Batch.DoesNotExist:
+ raise APIError('Batch does not exist')
+
+ batch_file = batch.files.filter(ready=True).order_by('-last_update_datetime').first()
+ if not batch_file:
+ raise APIError('No file available')
+
+ res = {}
+ batch_files = []
+ for i, batchfile in enumerate(batch.files.filter(ready=True).order_by('-last_update_datetime')):
+ batchfile_json = batchfile.to_json()
+ batchfile_json['url'] = request.build_absolute_uri(batchfile_json['url'])
+ if i == 0:
+ res['last_file'] = batchfile_json
+ batch_files.append(batchfile_json)
+
+ res['batch_files'] = batch_files
+ res.update(batch.to_json())
+ res['url'] = request.build_absolute_uri(res['url'])
+ return {
+ 'data': res
+ }
+
+ @endpoint(
+ perm='can_access', name='generate-batches',
+ post={
+ 'description': _('Generate batches'),
+ 'request_body': {
+ 'schema': {
+ 'application/json': GENERATE_BATCH_SCHEMA
+ }
+ }
+ }
+ )
+ def generate_batches(self, request, post_data):
+ batch_date = post_data.get('batch_date')
+ force = post_data.get('force', False)
+
+ def abs_url(data):
+ data['url'] = request.build_absolute_uri(data['url'])
+ return data
+
+ return {
+ 'data': [abs_url(batch.to_json()) for batch in self._generate_batches(batch_date, force)]
+
+ }
+
+ @transaction.atomic
+ def _generate_batches(self, batch_date=None, force=False):
+ target_date_obj = None
+ if batch_date is not None:
+ target_date_obj = parse_date(batch_date)
+ if target_date_obj is None:
+ raise APIError("Can't parse batch identifier")
+ force = True
+
+ code_app = 'TST003'
+ code_region = 'R04'
+ code_pays = 'FR'
+ devise = 'EUR'
+
+ def get_data(fields, field_name):
+ return fields[field_name] or ''
+
+ def write_row(writer, row):
+ writer.writerow([force_text(i) for i in row])
+
+ def add_target_batch(refs, target):
+ fd, temp_file_name = tempfile.mkstemp()
+ os.close(fd)
+ if six.PY3:
+ f = open(temp_file_name, 'w', encoding='utf-8')
+ refs[target] = (csv.writer(f, delimiter=';'), f, temp_file_name)
+ else:
+ f = open(temp_file_name, 'wb')
+ refs[target] = (csv.writer(f, delimiter=';', encoding='utf-8'), f, temp_file_name)
+
+ target_batches = {}
+ now = datetime.now().date()
+ for form in self._wcs_call():
+ avis = form['workflow']['fields']['avis_favorable_defavorable']
+ if not avis:
+ raise APIError('Missing avis')
+ if avis != 'Favorable':
+ continue
+ date_raw = form['workflow']['fields']['date_avis_favorable']
+ if not date_raw:
+ raise APIError("Missing date")
+ date_obj = parse_date(date_raw)
+ if date_obj is None:
+ raise APIError("Can't parse date")
+
+ # FOR TEST because no accepted form yet
+ # receipt_time = form['receipt_time']
+ # if not receipt_time:
+ # raise APIError("Missing receipt_time")
+ # date_obj = parse_datetime(receipt_time).date()
+ # if date_obj is None:
+ # raise APIError("Can't parse date")
+
+ if date_obj >= now:
+ continue
+ if target_date_obj and date_obj != target_date_obj:
+ continue
+ try:
+ batch = self.batches.get(batch_date=date_obj)
+ if force:
+ if batch not in target_batches:
+ add_target_batch(target_batches, batch)
+ else:
+ continue
+ except Batch.DoesNotExist:
+ batch = Batch.objects.create(resource=self, batch_date=date_obj)
+ add_target_batch(target_batches, batch)
+
+ # write data
+ form_fields = form['fields']
+ workflow_fields = form['workflow']['fields']
+ sequence = "%s-%s" % (code_region, form['display_id'])
+ period = 'MARS-VOLET2'
+ siren = form_fields['siren']
+ nom1 = form_fields['nom_responsable'] + form_fields['prenom_responsable']
+ nom2 = ''
+ nb_salaries = form_fields['nb_salaries']
+ rue = form_fields['numero_voie']
+ bp = get_data(form_fields, 'boite_postale')
+ cp = get_data(form_fields, 'code_postal')
+ ville = get_data(form_fields, 'ville')
+ iban = get_data(form_fields, 'iban')
+ montant = get_data(workflow_fields, 'montant_aide_volet_2')
+ nom_demandeur = form_fields['nom_demandeur']
+ prenom_demandeur = form_fields['prenom_demandeur']
+ qualite = ''
+ tel = get_data(form_fields, 'telephone_demandeur')
+ courriel = get_data(form_fields, 'courriel_demandeur')
+
+ write_row(
+ target_batches[batch][0],
+ [
+ code_app, sequence, period, siren, nom1, nom2, nb_salaries, rue,
+ bp, cp, ville, code_pays, code_region, iban, montant, devise, nom_demandeur,
+ prenom_demandeur, qualite, tel, courriel
+ ]
+ )
+
+ # create batch file objects
+ for batch, (csv_writer, f, temp_file_name) in target_batches.items():
+ f.close()
+ with open(temp_file_name, 'rb') as tf:
+ csv_filename = '%s.csv' % dateformat.format(batch.batch_date, 'Y-m-d')
+ batch_file = BatchFile.objects.create(batch=batch, csv_filename=csv_filename)
+ batch_file.csv_file.save(csv_filename, File(tf))
+ batch_file.ready = True
+ batch_file.save()
+ os.unlink(temp_file_name)
+
+ return [batch for batch in target_batches.keys()]
+
class Entreprise(models.Model):
@@ -516,3 +825,62 @@ class DSDossier(models.Model):
return None
return [{caption: get_cell(row, index) for caption, index in zip(caption, indexes)} for row in rows]
+
+
+class Batch(models.Model):
+
+ class Meta:
+ unique_together = (('resource', 'batch_date'),)
+ ordering = ('batch_date',)
+
+ resource = models.ForeignKey(
+ FSNReunionConnector, on_delete=models.CASCADE, related_name='batches'
+ )
+ batch_date = models.DateField()
+ last_update_datetime = models.DateTimeField(auto_now=True)
+
+ def to_json(self):
+ return {
+ 'batch_date': self.batch_date,
+ 'last_update_datetime': self.last_update_datetime,
+ 'url': self.get_absolute_url()
+ }
+
+ def get_absolute_url(self):
+ return reverse(
+ 'generic-endpoint',
+ kwargs={
+ 'connector': self.resource.get_connector_slug(),
+ 'slug': self.resource.slug,
+ 'endpoint': 'batch'
+ }
+ ) + '/%s/' % self.pk
+
+
+class BatchFile(models.Model):
+
+ batch = models.ForeignKey(
+ Batch, on_delete=models.CASCADE, related_name='files'
+ )
+ csv_file = models.FileField(upload_to=batch_csv_file_location)
+ csv_filename = models.CharField(max_length=256)
+ ready = models.BooleanField(default=False)
+ last_update_datetime = models.DateTimeField(auto_now=True)
+
+ def to_json(self):
+ return {
+ 'csv_filename': self.csv_filename,
+ 'ready': self.ready,
+ 'last_update_datetime': self.last_update_datetime,
+ 'url': self.get_absolute_url()
+ }
+
+ def get_absolute_url(self):
+ return reverse(
+ 'generic-endpoint',
+ kwargs={
+ 'connector': self.batch.resource.get_connector_slug(),
+ 'slug': self.batch.resource.slug,
+ 'endpoint': 'batchfile'
+ }
+ ) + '/%s/' % self.pk