lingo/lingo/invoicing/utils.py

# lingo - payment and billing system
# Copyright (C) 2022  Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import collections
import datetime

from django.db import transaction
from django.test.client import RequestFactory
from django.utils.timezone import localtime
from django.utils.translation import gettext_lazy as _

from lingo.agendas.chrono import get_check_status, get_subscriptions
from lingo.agendas.models import Agenda, CheckType
from lingo.invoicing.models import (
    DraftInvoice,
    DraftInvoiceLine,
    DraftJournalLine,
    InjectedLine,
    InvoicingError,
    Payer,
    Regie,
)
from lingo.pricing.models import AgendaPricing, AgendaPricingNotFound, PricingError


def get_agendas(pool):
    agendas_pricings = AgendaPricing.objects.filter(
        flat_fee_schedule=False, agendas__regie=pool.campaign.regie, agendas__in=pool.campaign.agendas.all()
    ).extra(
        where=['(date_start, date_end) OVERLAPS (%s, %s)'],
        params=[pool.campaign.date_start, pool.campaign.date_end],
    )
    return Agenda.objects.filter(pk__in=agendas_pricings.values('agendas')).order_by('pk')


def get_users_from_subscriptions(agendas, pool):
    users = {}
    for agenda in agendas:
        subscriptions = get_subscriptions(
            agenda_slug=agenda.slug,
            date_start=pool.campaign.date_start,
            date_end=pool.campaign.date_end,
        )
        for subscription in subscriptions:
            user_external_id = subscription['user_external_id']
            if user_external_id in users:
                continue
            users[user_external_id] = (subscription['user_first_name'], subscription['user_last_name'])
    return list(users.items())


def get_invoice_lines_for_user(
    agendas,
    agendas_pricings,
    user_external_id,
    user_first_name,
    user_last_name,
    pool,
    payer_data_cache,
):
    def get_agenda_pricing(agendas_pricings_for_agenda, date_event):
        # same logic as AgendaPricing.get_agenda_pricing
        for agenda_pricing in agendas_pricings_for_agenda:
            if agenda_pricing.date_start > date_event:
                continue
            if agenda_pricing.date_end <= date_event:
                continue
            return agenda_pricing
        raise AgendaPricingNotFound

    def get_cached_payer_data(request, payer_external_id, agenda_pricing=None, payer_data=None):
        if payer_external_id not in payer_data_cache:
            if agenda_pricing:
                # will raise a PricingError if payer_data can not be computed
                payer_data_cache[payer_external_id] = pool.campaign.regie.get_payer_data(
                    request, payer_external_id
                )
            elif payer_data:
                payer_data_cache[payer_external_id] = payer_data
        return payer_data_cache.get(payer_external_id) or {}

    if not agendas:
        return []

    agendas_by_slug = {a.slug: a for a in agendas}
    agendas_pricings_by_agendas = collections.defaultdict(list)
    for agenda_pricing in agendas_pricings:
        if agenda_pricing.flat_fee_schedule:
            continue
        for agenda in agenda_pricing.agendas.all():
            agendas_pricings_by_agendas[agenda.slug].append(agenda_pricing)

    # get check status for user_external_id, on agendas, for the period
    check_status_list = get_check_status(
        agenda_slugs=list(agendas_by_slug.keys()),
        user_external_id=user_external_id,
        date_start=pool.campaign.date_start,
        date_end=pool.campaign.date_end,
    )
    request = RequestFactory().get('/')  # XXX

    # get resolved errors in last pool for this user
    previous_pool = pool.campaign.pool_set.exclude(pk=pool.pk).order_by('created_at').last()
    resolved_errors = (
        {
            li.slug: li
            for li in DraftJournalLine.objects.filter(
                pool=previous_pool, status='error', user_external_id=user_external_id
            ).exclude(error_status='')
        }
        if previous_pool is not None
        else {}
    )

    # build lines from check status
    lines = []
    for check_status in check_status_list:
        serialized_event = check_status['event']
        event_date = datetime.datetime.fromisoformat(serialized_event['start_datetime']).date()
        event_slug = '%s@%s' % (serialized_event['agenda'], serialized_event['slug'])

        agenda = agendas_by_slug[serialized_event['agenda']]
        payer_external_id = _('unknown')
        payer_data = {}
        try:
            agenda_pricing = get_agenda_pricing(agendas_pricings_by_agendas.get(agenda.slug), event_date)
            payer_external_id = pool.campaign.regie.get_payer_external_id(request, user_external_id)
            payer_data = get_cached_payer_data(request, payer_external_id, agenda_pricing=agenda_pricing)
            pricing_data = agenda_pricing.get_pricing_data_for_event(
                request=request,
                agenda=agenda,
                event=serialized_event,
                check_status=check_status['check_status'],
                user_external_id=user_external_id,
                payer_external_id=payer_external_id,
            )
        except (PricingError, InvoicingError) as e:
            # AgendaPricingNotFound: can happen if pricing model defined only on a part of the requested period
            pricing_error = {
                'error': type(e).__name__,
                'error_details': e.details,
            }
            error_status = ''
            if resolved_errors.get(event_slug):
                error_status = resolved_errors[event_slug].error_status
            lines.append(
                DraftJournalLine(
                    event_date=event_date,
                    slug=event_slug,
                    label=serialized_event['label'],
                    amount=0,
                    user_external_id=user_external_id,
                    user_first_name=user_first_name,
                    user_last_name=user_last_name,
                    payer_external_id=payer_external_id,
                    payer_first_name=payer_data.get('first_name') or '',
                    payer_last_name=payer_data.get('last_name') or '',
                    payer_demat=payer_data.get('demat') or False,
                    payer_direct_debit=payer_data.get('direct_debit') or False,
                    event=serialized_event,
                    pricing_data=pricing_error,
                    status='warning' if isinstance(e, AgendaPricingNotFound) else 'error',
                    error_status=error_status,
                    pool=pool,
                )
            )
        else:
            # XXX log all context !
            lines.append(
                DraftJournalLine(
                    event_date=event_date,
                    slug=event_slug,
                    label=serialized_event['label'],
                    amount=pricing_data['pricing'],
                    user_external_id=user_external_id,
                    user_first_name=user_first_name,
                    user_last_name=user_last_name,
                    payer_external_id=payer_external_id,
                    payer_first_name=payer_data['first_name'],
                    payer_last_name=payer_data['last_name'],
                    payer_demat=payer_data['demat'],
                    payer_direct_debit=payer_data['direct_debit'],
                    event=serialized_event,
                    pricing_data=pricing_data,
                    status='success',
                    pool=pool,
                )
            )

    if pool.campaign.injected_lines != 'no':
        # fetch injected lines
        injected_lines = (
            InjectedLine.objects.filter(
                regie=pool.campaign.regie,
                event_date__lt=pool.campaign.date_end,
                user_external_id=user_external_id,
            )
            .exclude(
                # exclude already invoiced lines
                journalline__isnull=False
            )
            .exclude(
                # exclude lines used in another campaign
                pk__in=DraftJournalLine.objects.filter(from_injected_line__isnull=False)
                .exclude(pool__campaign=pool.campaign)
                .values('from_injected_line')
            )
            .order_by('event_date')
        )
        if pool.campaign.injected_lines == 'period':
            injected_lines = injected_lines.filter(
                event_date__gte=pool.campaign.date_start,
            )

        for injected_line in injected_lines:
            payer_external_id = injected_line.payer_external_id
            payer_data = {
                'first_name': injected_line.payer_first_name,
                'last_name': injected_line.payer_last_name,
                'demat': injected_line.payer_demat,
                'direct_debit': injected_line.payer_direct_debit,
            }
            payer_data = get_cached_payer_data(request, payer_external_id, payer_data=payer_data)
            lines.append(
                DraftJournalLine(
                    event_date=injected_line.event_date,
                    slug=injected_line.slug,
                    label=injected_line.label,
                    amount=injected_line.amount,
                    user_external_id=user_external_id,
                    user_first_name=user_first_name,
                    user_last_name=user_last_name,
                    payer_external_id=payer_external_id,
                    payer_first_name=payer_data['first_name'],
                    payer_last_name=payer_data['last_name'],
                    payer_demat=payer_data['demat'],
                    payer_direct_debit=payer_data['direct_debit'],
                    status='success',
                    pool=pool,
                    from_injected_line=injected_line,
                )
            )

    DraftJournalLine.objects.bulk_create(lines)

    return lines


def get_all_invoice_lines(agendas, users, pool):
    agendas_pricings = (
        AgendaPricing.objects.filter(flat_fee_schedule=False)
        .extra(
            where=['(date_start, date_end) OVERLAPS (%s, %s)'],
            params=[pool.campaign.date_start, pool.campaign.date_end],
        )
        .prefetch_related('agendas', 'pricing__criterias', 'pricing__categories')
    )

    lines = []
    payer_data_cache = {}
    for user_external_id, (user_first_name, user_last_name) in users:
        # generate lines for each user
        lines += get_invoice_lines_for_user(
            agendas=agendas,
            agendas_pricings=agendas_pricings,
            user_external_id=user_external_id,
            user_first_name=user_first_name,
            user_last_name=user_last_name,
            pool=pool,
            payer_data_cache=payer_data_cache,
        )
    return lines


def generate_invoices_from_lines(all_lines, pool):
    regie = pool.campaign.regie

    # regroup lines by payer_external_id (payer)
    lines = {}
    for line in all_lines:
        if line.status != 'success':
            # ignore lines in error
            continue
        if line.payer_external_id not in lines:
            lines[line.payer_external_id] = {
                'payer_first_name': line.payer_first_name,
                'payer_last_name': line.payer_last_name,
                'payer_demat': line.payer_demat,
                'payer_direct_debit': line.payer_direct_debit,
                'lines': [],
            }
        lines[line.payer_external_id]['lines'].append(line)

    # generate invoices by regie and by payer_external_id (payer)
    invoices = []
    for payer_external_id, payer_data in lines.items():
        invoice = DraftInvoice.objects.create(
            label=_('Invoice from %(start)s to %(end)s')
            % {
                'start': pool.campaign.date_start.strftime('%d/%m/%Y'),
                'end': (pool.campaign.date_end - datetime.timedelta(days=1)).strftime('%d/%m/%Y'),
            },
            date_publication=pool.campaign.date_publication,
            date_payment_deadline=pool.campaign.date_payment_deadline,
            date_due=pool.campaign.date_due,
            date_debit=pool.campaign.date_debit if payer_data['payer_direct_debit'] else None,
            regie=regie,
            payer_external_id=payer_external_id,
            payer_first_name=payer_data['payer_first_name'],
            payer_last_name=payer_data['payer_last_name'],
            payer_demat=payer_data['payer_demat'],
            payer_direct_debit=payer_data['payer_direct_debit'],
            pool=pool,
        )
        # regroup journal lines by user_external_id, status, check_type, check_type_group, pricing
        grouped_lines = collections.defaultdict(list)
        other_lines = []
        for line in payer_data['lines']:
            if line.pricing_data.get('booking_details', {}).get('status') == 'not-booked':
                # ignore not booked events
                continue
            if not line.event.get('primary_event'):
                # not a recurring event
                other_lines.append(line)
                continue
            key = (
                line.user_external_id,
                line.event['agenda'],
                line.event['primary_event'],
                line.pricing_data.get('booking_details', {}).get('status'),
                line.pricing_data.get('booking_details', {}).get('check_type'),
                line.pricing_data.get('booking_details', {}).get('check_type_group'),
                line.amount,
            )
            grouped_lines[key].append(line)
        for key, journal_lines in grouped_lines.items():
            journal_lines = sorted(journal_lines, key=lambda li: li.pk)
            first_line = journal_lines[0]
            slug = '-'.join(str(k) for k in key[1:])
            try:
                check_type = CheckType.objects.get(slug=key[4], group__slug=key[5], kind=key[3])
            except CheckType.DoesNotExist:
                check_type = None
            event_datetime = localtime(
                datetime.datetime.fromisoformat(journal_lines[0].event['start_datetime'])
            )
            event_time = event_datetime.time().isoformat()
            invoice_line = DraftInvoiceLine.objects.create(
                invoice=invoice,
                event_date=pool.campaign.date_start,
                slug=slug,
                label=first_line.label,
                quantity=len(journal_lines),
                unit_amount=first_line.amount,
                details={
                    'agenda': key[1],
                    'primary_event': key[2],
                    'status': key[3],
                    'check_type': key[4],
                    'check_type_group': key[5],
                    'check_type_label': check_type.label if check_type else key[4],
                    'dates': sorted(li.event_date for li in journal_lines),
                    'event_time': event_time,
                },
                user_external_id=first_line.user_external_id,
                user_first_name=first_line.user_first_name,
                user_last_name=first_line.user_last_name,
                payer_external_id=payer_external_id,
                payer_first_name=payer_data['payer_first_name'],
                payer_last_name=payer_data['payer_last_name'],
                payer_demat=payer_data['payer_demat'],
                payer_direct_debit=payer_data['payer_direct_debit'],
                pool=pool,
            )
            DraftJournalLine.objects.filter(pk__in=[li.pk for li in journal_lines]).update(
                invoice_line=invoice_line
            )
        for line in other_lines:
            invoice_line = DraftInvoiceLine.objects.create(
                invoice=invoice,
                event_date=line.event_date,
                slug=line.slug,
                label=line.label,
                quantity=1,
                unit_amount=line.amount,
                user_external_id=line.user_external_id,
                user_first_name=line.user_first_name,
                user_last_name=line.user_last_name,
                payer_external_id=payer_external_id,
                payer_first_name=payer_data['payer_first_name'],
                payer_last_name=payer_data['payer_last_name'],
                payer_demat=payer_data['payer_demat'],
                payer_direct_debit=payer_data['payer_direct_debit'],
                pool=pool,
            )
            line.invoice_line = invoice_line
            line.save()
        invoices.append(invoice)

    return invoices


def export_site(
    regies=True,
    payers=True,
):
    '''Dump site objects to JSON-dumpable dictionnary'''
    data = {}
    if regies:
        data['regies'] = [x.export_json() for x in Regie.objects.all()]
    if payers:
        data['payers'] = [x.export_json() for x in Payer.objects.all()]
    return data


def import_site(data):
    results = {
        key: collections.defaultdict(list)
        for key in [
            'regies',
            'payers',
        ]
    }

    with transaction.atomic():
        for cls, key in ((Payer, 'payers'), (Regie, 'regies')):
            objs = data.get(key, [])
            for obj in objs:
                created, obj = cls.import_json(obj)
                results[key]['all'].append(obj)
                if created:
                    results[key]['created'].append(obj)
                else:
                    results[key]['updated'].append(obj)
    return results