# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2018  Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import base64
import datetime
import json
import os
import random
import re
import zipfile
from ftplib import FTP
from urllib import parse as urlparse
from xml.etree import ElementTree as etree

import pdfrw
import pdfrw.findobjs
import zeep.exceptions
import zeep.helpers as zeep_helpers
from Cryptodome.Cipher import AES
from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.core.files.storage import default_storage
from django.core.signing import Signer
from django.db import models
from django.http import HttpResponse
from django.urls import reverse
from django.utils.encoding import force_str
from django.utils.timezone import now
from django.utils.translation import gettext_lazy as _

from passerelle.base.models import BaseResource
from passerelle.base.signature import sign_url
from passerelle.utils.api import endpoint
from passerelle.utils.jsonresponse import APIError, JSONEncoder


def cartads_file_location(instance, filename):
    return 'cartads_cs/%s/%s' % (instance.tracking_code, filename)


def key_value_of_stringstring(d):
    return {'KeyValueOfstringstring': [{'Key': x, 'Value': y} for x, y in d.items()]}


class CartaDSDataCache(models.Model):
    data_type = models.CharField(max_length=50)
    data_parameters = JSONField(default=dict)
    data_values = JSONField(default=dict)
    last_update_datetime = models.DateTimeField(auto_now=True)


class CartaDSFile(models.Model):
    tracking_code = models.CharField(max_length=20)
    id_piece = models.CharField(max_length=20)
    uploaded_file = models.FileField(upload_to=cartads_file_location)
    sent_to_cartads = models.DateTimeField(null=True)
    last_update_datetime = models.DateTimeField(auto_now=True)

    class Meta:
        ordering = ['id']


class CartaDSSubscriber(models.Model):
    name_id = models.CharField(max_length=32, null=True)


class CartaDSDossier(models.Model):
    email = models.CharField(max_length=256)
    name_id = models.CharField(max_length=32, null=True)
    tracking_code = models.CharField(max_length=20)
    commune_id = models.CharField(max_length=20)
    type_dossier_id = models.CharField(max_length=20)
    objet_demande_id = models.CharField(max_length=20, null=True)
    zip_ready = models.BooleanField(default=False)
    zip_sent = models.BooleanField(default=False)
    zip_ack_response = models.CharField(null=True, max_length=20)
    notification_url = models.URLField(null=True)
    notification_message = models.TextField(null=True)
    cartads_id_dossier = models.CharField(max_length=50, null=True)
    cartads_numero_dossier = models.CharField(max_length=50, null=True)
    cartads_cache_code_acces = models.CharField(max_length=200, null=True)
    cartads_cache_infos = JSONField(default=dict)
    cartads_steps_cache = JSONField(default=dict)
    last_update_datetime = models.DateTimeField(auto_now=True)
    subscribers = models.ManyToManyField(CartaDSSubscriber, blank=True)
    formdata_url = models.CharField(null=True, max_length=200)
    deleted = models.BooleanField(default=False)


class AbstractCartaDSCS(BaseResource):

    wsdl_base_url = models.URLField(
        _('WSDL Base URL'), help_text=_('ex: https://example.net/adscs/webservices/')
    )
    username = models.CharField(_('Username'), max_length=64)
    password = models.CharField(_('Password'), max_length=64)
    client_name = models.CharField(
        _('Client Name'),
        max_length=64,
        blank=True,
        null=True,
        help_text=_('Only useful in shared environments.'),
    )
    iv = models.CharField(_('Initialisation Vector'), max_length=16)
    secret_key = models.CharField(_('Secret Key'), max_length=16)
    ftp_server = models.CharField(_('FTP Server'), max_length=128)
    ftp_username = models.CharField(_('FTP Username'), max_length=64)
    ftp_password = models.CharField(_('FTP Password'), max_length=64)
    ftp_client_name = models.CharField(_('FTP Client Name'), max_length=64)

    class Meta:
        # it is abstract to make it possible for an external connector (@GL) to
        # reuse the full connector.
        abstract = True

    @property
    def wsdl_url(self):
        return self.get_wsdl_url()

    def get_wsdl_url(self, service_type='ServicePortail'):
        return self.wsdl_base_url + service_type + '.svc?singleWsdl'

    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)
        if CartaDSDataCache.objects.count() == 0:
            # don't wait for daily job to get initial data
            self.add_job('update_data_cache')

    def soap_client(self, **kwargs):
        client = super().soap_client(**kwargs)
        # fix URL that should have been changed by reverse proxy
        parsed_wsdl_address = urlparse.urlparse(client.service._binding_options['address'])
        parsed_real_address = urlparse.urlparse(self.wsdl_base_url)
        client.service._binding_options['address'] = urlparse.urlunparse(
            parsed_real_address[:2] + parsed_wsdl_address[2:]
        )
        return client

    def get_token(self):
        token_data = {
            'date': datetime.datetime.now().strftime('%d/%m/%Y %H:%M:%S'),
            'login': self.username,
            'password': self.password,
        }
        if self.client_name:
            token_data['client'] = self.client_name
        token_data_str = json.dumps(token_data)
        data_pad = AES.block_size - len(token_data_str) % AES.block_size
        aes = AES.new(self.secret_key.encode(), AES.MODE_CBC, self.iv.encode())
        token = aes.encrypt((token_data_str + (chr(data_pad) * data_pad)).encode())
        return force_str(base64.encodebytes(token)).replace('\n', '').rstrip('=')

    def check_status(self):
        self.soap_client().service.GetCommunes(self.get_token(), {})

    # description of common endpoint parameters
    COMMUNE_ID_PARAM = {'description': _('Identifier of collectivity'), 'example_value': '2'}
    TYPE_DOSSIER_ID_PARAM = {
        'description': _('Identifier of file type'),
        'example_value': 'CU',
    }
    OBJET_DEMANDE_ID_PARAM = {
        'description': _('Identifier of demand subject'),
        'example_value': '1',
    }
    TRACKING_CODE_PARAM = {
        'description': _('Unique identifier (ex: tracking code)'),
        'example_value': 'XCBTFRML',
    }
    PIECE_ID_PARAM = {
        'description': _('Identifier of single file item'),
    }
    UPLOAD_TOKEN_PARAM = {
        'description': _('Token for upload file'),
    }

    def get_cerfa_pdf(self, url):
        # method subclasses can override if the URL returned for PDF documents
        # by Cart@DS do not match reverse proxies, API managers, and stuff.
        return self.requests.get(url)

    def update_data_cache(self):
        client = self.soap_client()

        # communes
        resp = client.service.GetCommunes(self.get_token(), {})
        communes_cache, dummy = CartaDSDataCache.objects.get_or_create(data_type='communes')
        communes_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
        communes_cache.save()

        # types dossier
        types_dossier_ids = {}
        for commune in communes_cache.data_values['data']:
            resp = client.service.GetTypesDossier(self.get_token(), int(commune['id']), {})
            if resp is None:
                continue
            data_cache, dummy = CartaDSDataCache.objects.get_or_create(
                data_type='types_dossier', data_parameters={'commune_id': int(commune['id'])}
            )
            data_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
            types_dossier_ids.update({x['id']: True for x in data_cache.data_values['data']})
            data_cache.save()

        # objets_demande
        types_dossiers_objets_demandes_tuples = []
        objets_demande_ids = {}
        for type_dossier_id in types_dossier_ids:
            resp = client.service.GetObjetsDemande(self.get_token(), type_dossier_id)
            if resp is None:
                continue
            data_cache, dummy = CartaDSDataCache.objects.get_or_create(
                data_type='objets_demande', data_parameters={'type_dossier_id': type_dossier_id}
            )
            data_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
            objets_demande_ids.update({x['id']: True for x in data_cache.data_values['data']})
            types_dossiers_objets_demandes_tuples.extend(
                [(type_dossier_id, x['id']) for x in data_cache.data_values['data']]
            )
            data_cache.save()

        # liste_pdf
        pdfs_path = default_storage.path('public/cartads_cs/%s/documents' % self.slug)
        if not os.path.exists(pdfs_path):
            os.makedirs(pdfs_path)
        for type_compte in [1]:
            for type_dossier_id in types_dossier_ids:
                resp = client.service.GetListePdf(
                    self.get_token(), type_dossier_id, {'TypeCompteUtilisateur': type_compte}
                )
                if resp is None:
                    continue

                def format_cerfa_label(x):
                    try:
                        if x['Description']:
                            return '%(Nom)s: %(Description)s' % x
                    except KeyError:
                        pass
                    return '%(Nom)s' % x

                data_cache, dummy = CartaDSDataCache.objects.get_or_create(
                    data_type='liste_pdf',
                    data_parameters={
                        'type_dossier_id': type_dossier_id,
                        'type_compte': type_compte,
                    },
                )
                data_cache.data_values = {
                    'data': [
                        {
                            'id': x['Identifiant'],
                            'text': format_cerfa_label(x),
                            'url': x['UrlTelechargement'],
                        }
                        for x in resp or []
                    ]
                }

                for value in data_cache.data_values['data']:
                    filepath = os.path.join(default_storage.path(self.pdf_path(value)))
                    resp = self.get_cerfa_pdf(value['url'])
                    if resp.ok and resp.content.startswith(b'%PDF'):
                        with open(filepath, 'wb') as fd:
                            fd.write(resp.content)

                data_cache.save()

        # pieces
        for (type_dossier_id, objet_demande_id) in types_dossiers_objets_demandes_tuples:
            resp = client.service.GetPieces(self.get_token(), type_dossier_id, objet_demande_id)
            if resp is None:
                continue
            data_cache, dummy = CartaDSDataCache.objects.get_or_create(
                data_type='pieces',
                data_parameters={
                    'type_dossier_id': type_dossier_id,
                    'objet_demande_id': str(objet_demande_id),
                },
            )
            if resp is not None:
                data_cache.data_values = {
                    'data': [
                        {
                            'id': str(x['IdPiece']),
                            'text': x['Libelle'],
                            'description': x['Descriptif'],
                            'codePiece': x['CodePiece'],
                            'reglementaire': x['Reglementaire'],
                            'files': [],
                            'max_files': 6,
                        }
                        for x in resp
                    ]
                }
            data_cache.save()

    def get_dossier_steps(self, client, token, dossier):
        resp = client.service.GetEtapesDossier(token, dossier.cartads_id_dossier, [])
        steps = []
        for step in resp:
            step_dict = zeep_helpers.serialize_object(step)
            for key, value in step_dict.items():
                if isinstance(value, datetime.datetime):
                    step_dict[key] = value.strftime('%Y-%m-%dT%H:%M:%S')
            steps.append(step_dict)
        return steps

    def update_dossier_cache(self):
        client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceEtapeDossier'))
        client_suivi = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceSuiviNumerique'))
        client_dossier = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceDossier'))
        token = self.get_token()
        for dossier in CartaDSDossier.objects.filter(cartads_id_dossier__isnull=False, deleted=False):
            try:
                dossier.cartads_steps_cache = {'steps': self.get_dossier_steps(client, token, dossier)}
            except zeep.exceptions.Fault as e:
                if "n'existe pas" in str(e):
                    dossier.deleted = True
                    dossier.save()
                    continue
                self.logger.exception('error getting etapes of dossier (%s) (%s)', dossier.id, e)
            try:
                dossier.cartads_cache_code_acces = client_suivi.service.GetMotPasse(
                    self.get_token(), dossier.cartads_id_dossier
                )
            except zeep.exceptions.Fault as e:
                self.logger.exception('error getting access code (%s) (%s)', dossier.id, e)
            try:
                infos_dossier = client_dossier.service.GetInfosDossier(
                    self.get_token(), dossier.cartads_id_dossier
                )
                if infos_dossier:
                    # load(dump(...)) to serialize dates
                    dossier.cartads_cache_infos = json.loads(
                        json.dumps(zeep_helpers.serialize_object(infos_dossier), cls=JSONEncoder)
                    )
            except zeep.exceptions.Fault as e:
                self.logger.exception('error getting dossier infos (%s) (%s)', dossier.id, e)
            dossier.save()
            self.sync_subscribers_role(dossier)

    def hourly(self):
        super().hourly()
        self.update_dossier_cache()

    def daily(self):
        super().daily()
        self.update_data_cache()

    @endpoint(description=_('Get list of collectivities'))
    def communes(self, request):
        cache = CartaDSDataCache.objects.get(data_type='communes')
        return cache.data_values

    def get_commune_id(self, commune_name):
        for info in self.communes(request=None)['data']:
            if info['text'] == commune_name:
                return info['id']
        return None

    def get_commune_label(self, commune_id):
        for info in self.communes(request=None)['data']:
            if info['id'] == commune_id:
                return info['text']
        return None

    def get_type_dossier_label(self, commune_id, type_dossier_id):
        for info in self.types_dossier(request=None, commune_id=commune_id)['data']:
            if info['id'] == type_dossier_id:
                return info['text']
        return None

    @endpoint(
        description=_('Get list of file types'),
        parameters={
            'commune_id': COMMUNE_ID_PARAM,
            'filter': {
                'description': _('List of types to include (separated by commas)'),
                'example_value': 'CU,OP',
            },
        },
    )
    def types_dossier(self, request, commune_id, filter=None):
        cache = CartaDSDataCache.objects.get(
            data_type='types_dossier', data_parameters={'commune_id': int(commune_id)}
        )
        response = cache.data_values
        if filter:
            filter_list = filter.split(',')
            response['data'] = [x for x in response['data'] if x['id'] in filter_list]
        return response

    @endpoint(
        description=_('Get list of demand subjects'),
        parameters={'type_dossier_id': TYPE_DOSSIER_ID_PARAM},
    )
    def objets_demande(self, request, type_dossier_id):
        cache = CartaDSDataCache.objects.get(
            data_type='objets_demande', data_parameters={'type_dossier_id': type_dossier_id}
        )
        return cache.data_values

    @endpoint(
        description=_('Get list of CERFA documents'),
        parameters={
            'type_dossier_id': TYPE_DOSSIER_ID_PARAM,
            'type_compte': {'description': _('Type of account')},
        },
    )
    def liste_pdf(self, request, type_dossier_id, type_compte=1):
        cache = CartaDSDataCache.objects.get(
            data_type='liste_pdf',
            data_parameters={
                'type_dossier_id': type_dossier_id,
                'type_compte': type_compte,
            },
        )
        if request:  # point to local documents cache
            for pdf in cache.data_values['data']:
                pdf['url'] = request.build_absolute_uri(os.path.join(settings.MEDIA_URL, self.pdf_path(pdf)))
        return cache.data_values

    def pdf_path(self, pdf):
        if '*' in pdf['id']:  # cerfa
            filename = 'cerfa_%s.pdf' % pdf['id'].replace('*', '-')
        else:
            filename = '%s.pdf' % pdf['id']
        return os.path.join('public/cartads_cs', self.slug, 'documents', filename)

    @endpoint(
        perm='can_access',
        description=_('Get list of file items'),
        parameters={
            'type_dossier_id': TYPE_DOSSIER_ID_PARAM,
            'objet_demande_id': OBJET_DEMANDE_ID_PARAM,
            'tracking_code': TRACKING_CODE_PARAM,
            'demolitions': {
                'description': _('Include items for demolition work'),
                'example_value': 'false',
                'type': 'bool',
            },
        },
    )
    def pieces(self, request, type_dossier_id, objet_demande_id, tracking_code, demolitions=True):
        cache, dummy = CartaDSDataCache.objects.get_or_create(
            data_type='pieces',
            data_parameters={
                'type_dossier_id': type_dossier_id,
                'objet_demande_id': objet_demande_id,
            },
        )

        signer = Signer(salt='cart@ds_cs')
        upload_token = signer.sign(tracking_code)
        cerfa_pieces = [
            {
                'id': 'cerfa-%s-%s' % (type_dossier_id, objet_demande_id),
                'text': 'Cerfa rempli',
                'description': '',
                'codePiece': '',
                'reglementaire': True,
                'files': [],
                'max_files': 1,
                'section_start': 'Cerfa',
            },
            {
                'id': 'cerfa-autres-%s-%s' % (type_dossier_id, objet_demande_id),
                'text': 'Cerfa demandeurs complémentaires',
                'description': '',
                'codePiece': '',
                'reglementaire': False,
                'files': [],
                'max_files': 6,
            },
        ]
        pieces = cache.data_values['data'] if cache.data_values else []
        if demolitions is False:

            def is_demolition_piece(piece):
                if piece['reglementaire']:
                    return False
                for demolition_prefix in ('PCA', 'PCMIA'):
                    if re.match(r'^%s\d' % demolition_prefix, piece['codePiece']):
                        return True
                return False

            pieces = [x for x in pieces if not is_demolition_piece(x)]

        required_pieces = [x for x in pieces if x['reglementaire']]
        if required_pieces:
            required_pieces[0]['section_start'] = 'Pièces réglementaires'
        optional_pieces = [x for x in pieces if not x['reglementaire']]
        if optional_pieces:
            optional_pieces[0]['section_start'] = 'Pièces spécifiques'
        pieces = cerfa_pieces + required_pieces + optional_pieces
        known_files = CartaDSFile.objects.filter(tracking_code=tracking_code)
        for piece in pieces:

            if request:
                upload_url = request.build_absolute_uri(
                    '%supload/%s/%s/' % (self.get_absolute_url(), piece['id'], upload_token)
                )
            else:
                upload_url = None
            piece['files'] = [
                {
                    'url': upload_url,
                    'name': os.path.basename(x.uploaded_file.name),
                    'token': signer.sign(str(x.id)),
                    'id': x.id,
                }
                for x in known_files
                if x.id_piece == str(piece['id'])
            ]
            if len(piece['files']) < piece['max_files']:
                piece['files'].append({'url': upload_url})
        return {'data': pieces}

    @endpoint(
        perm='can_access',
        description=_('Check list of file items'),
        parameters={
            'type_dossier_id': TYPE_DOSSIER_ID_PARAM,
            'objet_demande_id': OBJET_DEMANDE_ID_PARAM,
            'tracking_code': TRACKING_CODE_PARAM,
        },
    )
    def check_pieces(self, request, type_dossier_id, objet_demande_id, tracking_code):
        pieces = self.pieces(request, type_dossier_id, objet_demande_id, tracking_code)
        result = True
        for piece in pieces['data']:
            if not piece['reglementaire']:
                continue
            if not [x for x in piece['files'] if x.get('name')]:
                result = False
                break
        return {'result': result}

    @endpoint(
        perm='can_access',
        description=_('Get list of additional file items'),
        parameters={
            'tracking_code': TRACKING_CODE_PARAM,
        },
    )
    def additional_pieces(self, request, tracking_code):
        client = self.soap_client(wsdl_url=self.get_wsdl_url('ServicePiece'))
        dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
        resp = client.service.GetPiecesDossierACompleter(self.get_token(), dossier.cartads_id_dossier)
        if resp is None:
            return {'data': []}

        signer = Signer(salt='cart@ds_cs')
        upload_token = signer.sign(tracking_code)

        pieces = [
            {
                'id': 'comp-%s-%s' % (x['IdDosPiece'], x['IdPiece']),
                'text': x['LibellePiece'],
                'description': x['Descriptif'],
                'codePiece': x['CodePiece'],
                'files': [],
                'max_files': 6,
            }
            for x in resp
        ]

        for piece in pieces:
            if request:
                upload_url = request.build_absolute_uri(
                    '%supload/%s/%s/' % (self.get_absolute_url(), piece['id'], upload_token)
                )
            else:
                upload_url = None
            piece['files'] = []
            piece['files'].append({'url': upload_url})
        return {'data': pieces}

    @endpoint(
        perm='can_access',
        description=_('Get list of DOC file items'),
        parameters={
            'tracking_code': TRACKING_CODE_PARAM,
        },
    )
    def doc_pieces(self, request, tracking_code):
        dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
        status = self.status(request, dossier.id)
        if status['status_label'] != 'Attente DOC':
            raise APIError('wrong status')

        signer = Signer(salt='cart@ds_cs')
        upload_token = signer.sign(tracking_code)
        pieces = [
            {
                'id': 'cerfa-doc',
                'text': 'CERFA',
                'description': '',
                'codePiece': '',
                'reglementaire': True,
                'files': [],
                'max_files': 1,
            },
        ]
        for piece in pieces:
            if request:
                upload_url = request.build_absolute_uri(
                    '%supload/%s/%s/' % (self.get_absolute_url(), piece['id'], upload_token)
                )
            else:
                upload_url = None
            piece['files'].append({'url': upload_url})
        return {'data': pieces}

    @endpoint(
        perm='can_access',
        description=_('Get list of DAACT file items'),
        parameters={
            'tracking_code': TRACKING_CODE_PARAM,
        },
    )
    def daact_pieces(self, request, tracking_code):
        dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
        status = self.status(request, dossier.id)
        if status['status_label'] != 'Attente DAACT':
            raise APIError('wrong status')

        client = self.soap_client(wsdl_url=self.get_wsdl_url('ServicePiece'))
        resp = client.service.GetPiecesDaact(self.get_token(), dossier.cartads_id_dossier)

        signer = Signer(salt='cart@ds_cs')
        upload_token = signer.sign(tracking_code)

        pieces = [
            {
                'id': 'cerfa-daact',
                'text': 'CERFA',
                'description': '',
                'codePiece': '',
                'reglementaire': True,
                'files': [],
                'max_files': 1,
            },
        ]

        pieces.extend(
            [
                {
                    'id': 'daact-%s' % x['IdPiece'],
                    'text': x['LibellePiece'],
                    'description': x['Descriptif'],
                    'codePiece': x['CodePiece'],
                    'files': [],
                    'max_files': 6,
                }
                for x in resp
            ]
        )

        for piece in pieces:
            if request:
                upload_url = request.build_absolute_uri(
                    '%supload/%s/%s/' % (self.get_absolute_url(), piece['id'], upload_token)
                )
            else:
                upload_url = None
            piece['files'] = []
            piece['files'].append({'url': upload_url})
        return {'data': pieces}

    @endpoint(
        methods=['post'],
        pattern=r'^(?P<id_piece>[\w-]+)/(?P<token>[\w:_-]+)/$',
        description=_('Upload a single document file'),
        parameters={
            'id_piece': PIECE_ID_PARAM,
            'token': UPLOAD_TOKEN_PARAM,
        },
    )
    def upload(self, request, id_piece, token, **kwargs):
        if not request.FILES.get('files[]'):
            # silently ignore request without files
            return []
        signer = Signer(salt='cart@ds_cs')
        tracking_code = signer.unsign(token)
        if id_piece.startswith('cerfa-'):
            try:
                pdf = pdfrw.PdfReader(request.FILES['files[]'])
                if not any(pdfrw.findobjs.find_objects(pdf, valid_subtypes=(pdfrw.PdfName.Form,))):
                    return [{'error': force_str(_('The CERFA should not be a scanned document.'))}]
            except (pdfrw.PdfParseError, ValueError):
                return [{'error': force_str(_('The CERFA should be a PDF file.'))}]
        else:
            if request.FILES['files[]'].content_type not in ('application/pdf', 'image/jpeg'):
                return [{'error': force_str(_('The file should be a PDF document or a JPEG image.'))}]
        if request.FILES['files[]'].size > 25 * 1024 * 1024:
            return [{'error': force_str(_('The file should not exceed 25MB.'))}]
        file_upload = CartaDSFile(
            tracking_code=tracking_code, id_piece=id_piece, uploaded_file=request.FILES['files[]']
        )
        file_upload.save()
        return [
            {
                'name': os.path.basename(file_upload.uploaded_file.name),
                'token': signer.sign(str(file_upload.id)),
            }
        ]

    @endpoint(
        methods=['post'],
        name='upload',
        pattern=r'^(?P<id_piece>[\w-]+)/(?P<token>[\w:_-]+)/(?P<file_upload>[\w:_-]+)/delete/$',
        description=_('Delete a single document file'),
        parameters={
            'id_piece': PIECE_ID_PARAM,
            'token': UPLOAD_TOKEN_PARAM,
            'file_upload': {
                'description': _('Signed identifier of single document upload'),
            },
        },
    )
    def upload_delete(self, request, id_piece, token, file_upload, **kwargs):
        # this cannot be verb DELETE as we have no way to set
        # Access-Control-Allow-Methods
        signer = Signer(salt='cart@ds_cs')
        CartaDSFile.objects.filter(id=signer.unsign(file_upload)).delete()
        return {'err': 0}

    @endpoint(
        perm='can_access',
        description=_('Validate and send a file'),
        parameters={
            'commune_id': COMMUNE_ID_PARAM,
            'type_dossier_id': TYPE_DOSSIER_ID_PARAM,
            'objet_demande_id': OBJET_DEMANDE_ID_PARAM,
            'tracking_code': TRACKING_CODE_PARAM,
            'email': {
                'description': _('Email of requester'),
            },
            'name_id': {
                'description': _('UUID of requester'),
            },
            'formdata_url': {
                'description': _('URL of user form'),
            },
        },
    )
    def send(
        self,
        request,
        commune_id,
        type_dossier_id,
        objet_demande_id,
        tracking_code,
        email,
        name_id=None,
        formdata_url=None,
    ):
        dossier = CartaDSDossier(
            commune_id=commune_id,
            type_dossier_id=type_dossier_id,
            objet_demande_id=objet_demande_id,
            tracking_code=tracking_code,
            email=email,
            formdata_url=formdata_url,
        )
        dossier.save()
        signer = Signer(salt='cart@ds_cs/dossier')
        notification_base_url = reverse(
            'generic-endpoint',
            kwargs={'connector': self.get_connector_slug(), 'slug': self.slug, 'endpoint': 'notification'},
        )
        dossier.notification_url = request.build_absolute_uri(
            notification_base_url + '/%s/' % signer.sign(str(dossier.id))
        )
        dossier.save()
        if name_id:
            dossier.subscribers.add(CartaDSSubscriber.objects.get_or_create(name_id=name_id)[0])
        self.add_job('pack', dossier_id=dossier.id)
        return {'err': 0, 'dossier_id': dossier.id, 'tracking_code': dossier.tracking_code}

    def pack(self, dossier_id):
        dossier = CartaDSDossier.objects.get(id=dossier_id)
        zip_filename = os.path.join(default_storage.path('cartads_cs'), '%s.zip' % dossier.tracking_code)
        with zipfile.ZipFile(zip_filename, mode='w') as zip_file:
            liste_pdf = self.liste_pdf(None, dossier.type_dossier_id)
            cerfa_id = liste_pdf['data'][0]['id']
            for cerfa in liste_pdf['data']:
                if cerfa['id'] == 'AUTRES_DEMANDEURS':
                    continue
                cerfa_id = cerfa['id']
                break
            cerfa_id = cerfa_id.replace('*', '-')
            pieces = self.pieces(
                None, dossier.type_dossier_id, dossier.objet_demande_id, dossier.tracking_code
            )
            for piece in pieces['data']:
                cnt = 1
                for file in piece['files']:
                    if not file.get('id'):
                        continue
                    cartads_file = CartaDSFile.objects.get(id=file['id'])
                    if piece['id'] == 'cerfa-%s-%s' % (dossier.type_dossier_id, dossier.objet_demande_id):
                        zip_file.write(cartads_file.uploaded_file.path, '%s.pdf' % cerfa_id)
                    elif piece['id'].startswith('cerfa-autres-'):
                        zip_file.write(
                            cartads_file.uploaded_file.path,
                            'Fiches_complementaires/Cerfa_autres_demandeurs_%d.pdf' % cnt,
                        )
                    else:
                        zip_file.write(
                            cartads_file.uploaded_file.path,
                            'Pieces/%s-%s%s%s'
                            % (
                                piece['id'],
                                piece['codePiece'],
                                cnt,
                                os.path.splitext(cartads_file.uploaded_file.path)[-1],
                            ),
                        )
                    cnt += 1
        dossier.zip_ready = True
        dossier.save()
        self.add_job('send_to_cartads', dossier_id=dossier.id)

    def upload_zip(self, zip_filename):
        ftp = FTP(self.ftp_server)
        ftp.login(self.ftp_username, self.ftp_password)
        ftp.cwd(self.ftp_client_name)
        with open(zip_filename, 'rb') as fd:
            ftp.storbinary('STOR %s' % os.path.basename(zip_filename), fd)
            ftp.quit()

    def send_to_cartads(self, dossier_id):
        dossier = CartaDSDossier.objects.get(id=dossier_id)
        zip_filename = os.path.join(default_storage.path('cartads_cs'), '%s.zip' % dossier.tracking_code)
        self.upload_zip(zip_filename)

        client = self.soap_client()
        resp = client.service.NotifierDepotDossier(
            self.get_token(),
            dossier.commune_id,
            dossier.type_dossier_id,
            os.path.basename(zip_filename),
            dossier.email,
            key_value_of_stringstring(
                {
                    'NotificationMailDemandeur': '0',
                    'IdDossierExterne': 'publik-%s-%s' % (dossier.id, dossier.tracking_code),
                    'NumeroDossierExterne': 'publik-%s-%s' % (dossier.id, dossier.tracking_code),
                    'TraitementImmediat': '0',
                    'UrlNotification': dossier.notification_url,
                }
            ),
        )
        dossier.zip_sent = True
        dossier.zip_ack_response = str(resp)
        dossier.save()
        CartaDSFile.objects.filter(tracking_code=dossier.tracking_code).update(sent_to_cartads=now())
        self.sync_subscribers_role(dossier)

    @endpoint(
        pattern=r'^(?P<signed_dossier_id>[\w:_-]+)/$',
        methods=['post'],
        description=_('Notification of file processing by Cart@DS CS'),
        parameters={
            'signed_dossier_id': {'description': _('Signed identifier of file')},
        },
    )
    def notification(self, request, signed_dossier_id):
        signer = Signer(salt='cart@ds_cs/dossier')
        dossier_id = signer.unsign(signed_dossier_id)
        dossier = CartaDSDossier.objects.get(id=dossier_id)
        dossier.notification_message = request.POST['notification']
        notification = etree.fromstring(dossier.notification_message.encode('utf-8'))
        if notification.find('Succes').text == 'true':
            dossier.cartads_id_dossier = notification.find(
                'InformationsComplementaires/IdDossierCartads'
            ).text
            dossier.cartads_numero_dossier = notification.find(
                'InformationsComplementaires/NumeroDossier'
            ).text
            self.sync_subscribers_role(dossier)
        dossier.save()
        return HttpResponse('ok', content_type='text/plain')

    @endpoint(
        perm='can_access',
        description=_('Send requested additional file items'),
        parameters={
            'tracking_code': TRACKING_CODE_PARAM,
        },
    )
    def send_additional_pieces(self, request, tracking_code):
        dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
        self.add_job('send_additional_pieces_to_cartads', dossier_id=dossier.id)
        return {'err': 0, 'dossier_id': dossier.id}

    def send_additional_pieces_to_cartads(self, dossier_id):
        dossier = CartaDSDossier.objects.get(id=dossier_id)
        client = self.soap_client(wsdl_url=self.get_wsdl_url('ServicePiece'))
        resp = client.service.GetPiecesDossierACompleter(self.get_token(), dossier.cartads_id_dossier)
        pieces = [
            {
                'id': 'comp-%s-%s' % (x['IdDosPiece'], x['IdPiece']),
                'idPiece': x['IdPiece'],
                'codePiece': x['CodePiece'],
            }
            for x in resp
        ]

        client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceDocumentation'))

        for piece_type in pieces:
            for i, piece in enumerate(
                CartaDSFile.objects.filter(tracking_code=dossier.tracking_code, id_piece=piece_type['id'])
            ):
                if piece.sent_to_cartads:
                    continue
                id_dos_piece = piece.id_piece.split('-')[1]
                filename = '%s-%s%s%s' % (
                    piece_type['idPiece'],
                    piece_type['codePiece'],
                    '%03d' % (i + 1),
                    os.path.splitext(piece.uploaded_file.name)[-1],
                )
                content = piece.uploaded_file.read()
                try:
                    resp = client.service.UploadFile(
                        FileByteStream=content,
                        _soapheaders={
                            'IdDossier': dossier.cartads_id_dossier,
                            'NomFichier': filename,
                            'Length': piece.uploaded_file.size,
                            'Token': self.get_token(),
                            'InformationsComplementaires': key_value_of_stringstring(
                                {'idDosPiece': id_dos_piece}
                            ),
                        },
                    )
                except zeep.exceptions.Fault as e:
                    self.logger.exception('error pushing file item %d (%s)', piece.id, e)
                    continue
                else:
                    assert resp is None
                    piece.sent_to_cartads = now()
                    piece.save()

    @endpoint(
        perm='can_access',
        description=_('Send DOC file items'),
        parameters={
            'tracking_code': TRACKING_CODE_PARAM,
        },
    )
    def send_doc_pieces(self, request, tracking_code):
        dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
        self.add_job('send_doc_pieces_to_cartads', dossier_id=dossier.id)
        return {'err': 0, 'dossier_id': dossier.id}

    def send_doc_pieces_to_cartads(self, dossier_id):
        dossier = CartaDSDossier.objects.get(id=dossier_id)
        client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceDocumentation'))
        pieces = CartaDSFile.objects.filter(
            tracking_code=dossier.tracking_code, id_piece='cerfa-doc', sent_to_cartads__isnull=True
        )
        assert pieces.count() == 1
        piece = pieces[0]

        content = piece.uploaded_file.read()
        try:
            resp = client.service.UploadFile(
                FileByteStream=content,
                _soapheaders={
                    'IdDossier': dossier.cartads_id_dossier,
                    'NomFichier': 'cerfa-doc.pdf',
                    'Length': piece.uploaded_file.size,
                    'Token': self.get_token(),
                    'InformationsComplementaires': key_value_of_stringstring(
                        {'docDaact': 'doc', 'renameFile': 'true'},
                    ),
                },
            )
        except zeep.exceptions.Fault as e:
            self.logger.exception('error pushing file item %d (%s)', piece.id, e)
        else:
            assert resp is None
            piece.sent_to_cartads = now()
            piece.save()

    @endpoint(
        perm='can_access',
        description=_('Send DAACT file items'),
        parameters={
            'tracking_code': TRACKING_CODE_PARAM,
        },
    )
    def send_daact_pieces(self, request, tracking_code):
        dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
        self.add_job('send_daact_pieces_to_cartads', dossier_id=dossier.id)
        return {'err': 0, 'dossier_id': dossier.id}

    def send_daact_pieces_to_cartads(self, dossier_id):
        dossier = CartaDSDossier.objects.get(id=dossier_id)
        pieces = self.daact_pieces(None, dossier.tracking_code)['data']

        client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceDocumentation'))
        for piece_type in pieces:
            for piece in CartaDSFile.objects.filter(
                tracking_code=dossier.tracking_code, sent_to_cartads__isnull=True, id_piece=piece_type['id']
            ):
                content = piece.uploaded_file.read()
                try:
                    infos = {
                        'renameFile': 'true',
                    }
                    if piece.id_piece == 'cerfa-daact':
                        infos['docDaact'] = 'daact'
                        filename = 'cerfa-daact.pdf'
                    else:
                        infos['docDaact'] = 'pieceDaact'
                        infos['idPieceDaact'] = piece.id_piece.split('-', 1)[-1]
                        filename = '%s%s' % (
                            piece_type['codePiece'],
                            os.path.splitext(piece.uploaded_file.name)[-1],
                        )
                    resp = client.service.UploadFile(
                        FileByteStream=content,
                        _soapheaders={
                            'IdDossier': dossier.cartads_id_dossier,
                            'NomFichier': filename,
                            'Length': piece.uploaded_file.size,
                            'Token': self.get_token(),
                            'InformationsComplementaires': key_value_of_stringstring(infos),
                        },
                    )
                except zeep.exceptions.Fault as e:
                    self.logger.exception('error pushing daact file item %d (%s)', piece.id, e)
                else:
                    assert resp is None
                    piece.sent_to_cartads = now()
                    piece.save()

    def get_file_status(self, dossier):
        response = {}
        if dossier.deleted:
            status_id = 'deleted'
            status_label = _('Deleted')
        elif dossier.cartads_id_dossier:
            if dossier.cartads_steps_cache:
                steps = dossier.cartads_steps_cache['steps']
            else:
                client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceEtapeDossier'))
                steps = self.get_dossier_steps(client, self.get_token(), dossier)
                dossier.cartads_steps_cache['steps'] = steps
                dossier.save()
            steps.sort(key=lambda x: x['DateReference'])
            status_id = 'cartads-%s' % steps[-1]['IdEtape']
            status_label = steps[-1]['LibelleEtape']
            response['extra'] = {}
            for key in steps[-1]:
                response['extra'][key] = steps[-1][key]
            response['cartads_reference_dossier'] = dossier.cartads_numero_dossier
            response['cartads_code_acces'] = dossier.cartads_cache_code_acces
        elif dossier.notification_message:  # but not dossier id -> error
            status_id = 'refused'
            notification = etree.fromstring(dossier.notification_message.encode('utf-8'))
            error = notification.find('InformationsComplementaires/MessageErreur').text
            status_label = _('File refused (%s)') % error
        elif dossier.zip_sent:
            status_id = 'zip-sent'
            status_label = _('File sent')
            if dossier.zip_ack_response == 'False':
                status_id = 'zip-not-considered'
                status_label = _('File not considered')
        elif dossier.zip_ready:
            status_id = 'zip-ready'
            status_label = _('File ready to be sent')
        else:
            status_id = 'pending'
            status_label = _('Pending')
        response.update({'status_id': status_id, 'status_label': status_label})
        return response

    @endpoint(
        perm='can_access',
        description=_('Get status of file'),
        parameters={
            'dossier_id': {
                'description': _('Identifier of file'),
            }
        },
    )
    def status(self, request, dossier_id):
        dossier = CartaDSDossier.objects.get(id=dossier_id)
        return self.get_file_status(dossier)

    @endpoint(
        perm='can_access',
        description=_('Get list of files attached to user'),
        parameters={
            'name_id': {'description': _('UUID of requester'), 'example_value': '3eb56fc'},
            'status': {
                'description': _('File Status'),
                'example_value': 'Attente DOC',
            },
        },
    )
    def files(self, request, name_id, status=None):
        files = CartaDSDossier.objects.filter(subscribers__name_id__in=[name_id])
        if status:
            files = [
                x
                for x in files
                if self.get_file_status(x).get('status_id') == status
                or self.get_file_status(x).get('status_label') == status
            ]

        def get_date(dossier):
            if dossier.cartads_cache_infos and dossier.cartads_cache_infos['DateDepot']:
                return dossier.cartads_cache_infos['DateDepot']
            return ''

        files = list(files)
        files.sort(key=get_date)

        return {
            'data': [
                {
                    'id': str(x.id),
                    'text': x.cartads_numero_dossier,
                    'tracking_code': x.tracking_code,
                    'status': self.get_file_status(x),
                    'commune_label': self.get_commune_label(x.commune_id),
                    'type_dossier_label': self.get_type_dossier_label(x.commune_id, x.type_dossier_id),
                    'formdata_url': x.formdata_url,
                    'cartads_infos': x.cartads_cache_infos,
                }
                for x in files
            ]
        }

    @endpoint(
        perm='can_access',
        description=_('Join dossier'),
        parameters={
            'name_id': {'description': _('UUID of requester'), 'example_value': '3eb56fc'},
            'dossier_number': {
                'description': _('Dossier Number'),
                'example_value': 'PC 069 012 23 45678',
            },
            'dossier_password': {
                'description': _('Dossier Password'),
                'example_value': '5A3E36FE-80D3-45E5-9323-7415E04D3B14',
            },
            'formdata_url': {
                'description': _('URL of user form'),
            },
        },
    )
    def join(self, request, name_id, dossier_number, dossier_password, formdata_url=None):
        client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceSuiviNumerique'))
        try:
            resp = client.service.ActiverServiceSuiviNumerique(
                self.get_token(), dossier_number, dossier_password
            )
        except zeep.exceptions.Fault as e:
            self.logger.error('error joining dossier %s (%s)', dossier_number, e)
            raise APIError('error joining dossier (wrong password?)')
        id_dossier = int(resp)
        dossier, created = CartaDSDossier.objects.get_or_create(cartads_id_dossier=id_dossier)
        if created:
            dossier.cartads_numero_dossier = dossier_number
            client_dossier = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceRechercheDossier'))
            infos = client_dossier.service.GetInfosDossier(self.client_name, id_dossier)
            dossier.type_dossier_id = infos['CoTypeDossier']
            dossier.commune_id = self.get_commune_id(infos['Commune'])
            dossier.formdata_url = formdata_url
            CHARS = 'BCDFGHJKLMNPQRSTVWXZ'
            r = random.SystemRandom()
            dossier.tracking_code = 'A-' + ''.join([r.choice(CHARS) for x in range(8)])
            dossier.save()

        dossier.subscribers.add(CartaDSSubscriber.objects.get_or_create(name_id=name_id)[0])
        self.sync_subscribers_role(dossier)
        return {
            'err': 0,
            'dossier_id': dossier.id,
            'formdata_url': dossier.formdata_url,
            'tracking_code': dossier.tracking_code,
        }

    @endpoint(
        perm='can_access',
        description=_('Unsubscribe from dossier'),
        parameters={
            'name_id': {'description': _('UUID of requester'), 'example_value': '3eb56fc'},
            'dossier_number': {
                'description': _('Dossier Number'),
                'example_value': 'PC 069 012 23 45678',
            },
        },
    )
    def unsubscribe(self, request, name_id, dossier_number):
        try:
            dossier = CartaDSDossier.objects.get(cartads_numero_dossier=dossier_number)
        except CartaDSDossier.DoesNotExist:
            raise APIError('dossier does not exist')
        try:
            subscriber = CartaDSSubscriber.objects.get(name_id=name_id)
        except CartaDSSubscriber.DoesNotExist:
            raise APIError('subscriber does not exist')
        if subscriber not in dossier.subscribers.all():
            raise APIError('subscriber not subscribed to that dossier')
        dossier.subscribers.remove(subscriber)
        self.sync_subscribers_role(dossier)
        return {'err': 0, 'dossier_id': dossier.id}

    def sync_subscribers_role(self, dossier):
        if not getattr(settings, 'KNOWN_SERVICES', {}).get('authentic'):
            return
        idp_service = list(settings.KNOWN_SERVICES['authentic'].values())[0]
        # sync subscribers with an authentic role, this can fail and it will
        # be retried again later.
        role_api_url = sign_url(
            urlparse.urljoin(
                idp_service['url'], 'api/roles/?get_or_create=slug&orig=%s' % idp_service.get('orig')
            ),
            key=idp_service.get('secret'),
        )
        response = self.requests.post(
            role_api_url,
            json={
                'name': 'Suivi Cart@DS (%s)' % dossier.id,
                'slug': '_cartads_%s' % dossier.id,
            },
        )
        if response.status_code != 200:
            return
        try:
            role_uuid = response.json()['uuid']
        except (KeyError, TypeError, ValueError):
            return
        role_api_url = sign_url(
            urlparse.urljoin(
                idp_service['url'],
                'api/roles/%s/relationships/members/?orig=%s' % (role_uuid, idp_service.get('orig')),
            ),
            key=idp_service.get('secret'),
        )
        response = self.requests.patch(
            role_api_url, json={'data': [{'uuid': x.name_id} for x in dossier.subscribers.all()]}
        )


class CartaDSCS(AbstractCartaDSCS):
    category = _('Misc')

    class Meta:
        verbose_name = 'Cart@DS CS'
