nanterre: détecte et supprime les fiches inactives (fixes #28080)

debian/control

@@ -17,6 +17,7 @@ Depends: ${misc:Depends}, adduser,
     python-psycopg2,
     python-memcache,
     python-django-mellon,
+    python-cached-property,
     uwsgi
 Recommends: nginx, postgresql, memcached
 Description: Maintain a graph of objects

setup.py

@@ -108,6 +108,7 @@ setup(
         'python-dateutil',
         'django-admin-rangefilter',
         'requests',
+        'cached-property',
     ],
     zip_safe=False,
     cmdclass={

zoo/zoo_nanterre/apps.py

@@ -95,6 +95,13 @@ class ZooNanterreConfig(AppConfig):
                 kwargs={'model_admin': model_admin},
                 name='synchronize-federations' + desc['name'],
             ))
+        urls.append(url(
+            r'^inactive/',
+            model_admin.admin_site.admin_view(
+                getattr(views, 'inactive_index')),
+            kwargs={'model_admin': model_admin},
+            name='inactive-index',
+        ))
         return urls
 
     def post_migrate(self, *args, **kwargs):

zoo/zoo_nanterre/inactivity.py

@@ -0,0 +1,123 @@
+# zoo - versatile objects management
+# Copyright (C) 2016  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+import operator
+try:
+    from functools import reduce
+except ImportError:
+    pass
+
+from django.db.models.query import Q
+from django.utils.timezone import now
+
+from cached_property import cached_property
+
+from zoo.zoo_data.models import Entity
+
+from . import utils
+
+
+class Inactivity(object):
+    def __init__(self, child_delay=365, adult_delay=365):
+        self.child_delay = child_delay
+        self.adult_delay = adult_delay
+
+    @property
+    def child_threshold(self):
+        return now() - datetime.timedelta(days=self.child_delay)
+
+    @property
+    def adult_threshold(self):
+        return now() - datetime.timedelta(days=self.adult_delay)
+
+    def exclude_newer_than_threshold(self, qs, threshold):
+        return qs.exclude(
+            created__created__gte=threshold,
+            modified__created__gte=threshold,
+            log__timestamp__gte=threshold)
+
+    @property
+    def query_no_federation(self):
+        queries = []
+
+        for app_id in utils.get_applications(rsu_ws_url=True):
+            query = Q(**{'content__cles_de_federation__%s__isnull' % app_id: True})
+            query |= Q(**{'content__cles_de_federation__%s__exact' % app_id: ''})
+            queries.append(query)
+        return reduce(operator.__and__, queries)
+
+    def filter_no_federation(self, qs):
+        return qs.filter(self.query_no_federation)
+
+    @property
+    def entities(self):
+        qs = Entity.objects.all()
+        # prefetch to optimize accesors to spouses and siblings
+        qs = qs.prefetch_related(
+            'left_relations__schema', 'left_relations__right',
+            'right_relations__schema', 'right_relations__left',
+            'right_relations__left__left_relations__left',
+            'right_relations__left__left_relations__schema',
+        )
+        return qs
+
+    @property
+    def children(self):
+        return self.entities.filter(content__statut_legal='mineur')
+
+    @property
+    def adults(self):
+        return self.entities.filter(content__statut_legal='majeur')
+
+    @cached_property
+    def deletable_children(self):
+        potent = self.exclude_newer_than_threshold(self.filter_no_federation(self.children), self.child_threshold)
+        potent_ids = potent.values_list('id', flat=True)
+
+        def filter_siblings_are_potent():
+            for child in potent:
+                for sibling in utils.fratrie(child):
+                    if sibling.id not in potent_ids:
+                        break
+                else:
+                    yield child.id
+        potent2_ids = list(filter_siblings_are_potent())
+
+        return potent.filter(id__in=potent2_ids)
+
+    @cached_property
+    def deletable_adults(self):
+        deletable_children_ids = self.deletable_children.values_list('id', flat=True)
+        potent = self.exclude_newer_than_threshold(self.filter_no_federation(self.adults), self.adult_threshold)
+
+        def filter_children_are_deletable():
+            for adult in potent:
+                for enfant, rele in utils.enfants(adult):
+                    if enfant.id not in deletable_children_ids:
+                        break
+                else:
+                    yield adult
+        potent2 = list(filter_children_are_deletable())
+        potent2_ids = [adult.id for adult in potent2]
+
+        def filter_spouse_is_deletable():
+            for adult in potent2:
+                conjoint = utils.conjoint(adult)[0]
+                if conjoint and conjoint.id not in potent2_ids:
+                    continue
+                yield adult.id
+        return potent.filter(id__in=filter_spouse_is_deletable())

zoo/zoo_nanterre/models.py

@@ -1,3 +1,19 @@
+# zoo - versatile objects management
+# Copyright (C) 2016  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 from django.utils.translation import ugettext_lazy as _
 from django.contrib.postgres.fields import JSONField
 from django.db import models

zoo/zoo_nanterre/templates/admin/index.html

@@ -61,6 +61,17 @@
           <td>&nbsp;</td>
         </tr>
       {% endif %}
+      {% if perms.zoo_data.action2_entity %}
+        <tr>
+          <th scope="row">
+            <a href="{% url "admin:inactive-index" %}">
+              Fiches inactives
+            </a>
+          </th>
+          <td>&nbsp;</td>
+          <td>&nbsp;</td>
+        </tr>
+      {% endif %}
     </table>
   </div>
 </div>

zoo/zoo_nanterre/templates/admin/zoo_data/entity/inactive_index.html

@@ -0,0 +1,72 @@
+{% extends "admin/base_site.html" %}
+{% load i18n admin_urls static %}
+
+{% block extrastyle %}
+  {{ block.super }}
+  <link rel="stylesheet" type="text/css" href="{% static "admin/css/changelists.css" %}" />
+{% endblock %}
+
+{% block coltype %}flex{% endblock %}
+
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+  <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+  &rsaquo; Fiches inactives
+</div>
+{% endblock %}
+
+{% block content %}
+<div id="content-main">
+  <ul class="object-tools">
+    <li><a href="?recompute">Recalculer</a></li>
+    <li><a href="?csv">Export CSV</a></li>
+    <li><a href="#" onclick="document.getElementById('delete-button').click()">Supprimer</a></li>
+    <form method="post" action="?delete" style="display: none">{% csrf_token %}<button id="delete-button">Supprimer</button></form>
+  </ul>
+  <div style="float: right">
+      <p>{{ fiches|length }} fiches. Dernier calcul le {{ timestamp }}. {% if duration %} Durée {{ duration }} secondes.{% endif %}{% if queries %}{{ queries }} requêtes.{% endif %}</p>
+      <h3>Explication du calcul</h3>
+      <ul>
+          <li>Une fiche sans activité est une fiche dont la dernière date de modification et les dernières lignes du journal n'ont pas bougé depuis un certain nombre de jours</li>
+          <li>Les fiches enfants sans activité depuis {{ child_delay }} jours sont considérées inactives.</li>
+          <li>Les fiches adultes sans activité depuis {{ adult_delay }} jours sont considérées inactives.</li>
+          <li>Les fratries (tous les enfants partageant au moins un parent commun) inactives et sans fédérations sont à supprimer.</li>
+          <li>Les adultes sans liens matrimoniaux, inactifs, sans fédérations et dont tous les enfants sont à supprimer sont à supprimer.</li>
+          <li>Les couples inactifs, sans fédérations et dont tous les enfants sont à supprimer sont à supprimer.</li>
+      </ul>
+      <h3>Configuration</h3>
+      <ul>
+          <li>Le délai d'inactivité en jours pour les fiches enfants est configuré par le setting <tt>ZOO_NANTERRE_INACTIVITY_CHILD_DELAY</tt></li>
+          <li>Le délai d'inactivité en jours pour les fiches adultes est configuré par le setting <tt>ZOO_NANTERRE_INACTIVITY_ADULT_DELAY</tt></li>
+      </ul>
+  </div>
+  <table id="result-list">
+    {% comment %}Fiches inactives{% endcomment %}
+    <thead>
+      <tr>
+        <th>Identifiant RSU</th>
+        <th>Prénom</th>
+        <th>Nom d'usage</th>
+        <th>Nom de naissance</th>
+        <th>Date de naissance</th>
+        <th>Statut légal</th>
+        <th>Âge</th>
+      </tr>
+     </thead>
+     <tbody>
+       {% for fiche in fiches %}
+         <tr>
+           <td>{{ fiche.id }}</td>
+           <td>{{ fiche.prenoms }}</td>
+           <td>{{ fiche.nom_d_usage }}</td>
+           <td>{{ fiche.nom_de_naissance }}</td>
+           <td>{{ fiche.date_de_naissance }}</td>
+           <td>{{ fiche.statut_legal }}</td>
+           <td>{{ fiche.age }}</td>
+         </tr>
+       {% endfor %}
+     </tbody>
+  </table>
+</div>
+{% endblock %}

zoo/zoo_nanterre/utils.py

@@ -20,6 +20,8 @@ from __future__ import print_function
 import six
 import functools
 import uuid
+import io
+import csv
 
 import sys
 import re
@@ -39,8 +41,11 @@ from django.db import connection
 from django.db.models import Q, F, Value, ExpressionWrapper, CharField, When, Case
 from django.db.models.functions import Least, Greatest, Coalesce, Concat
 from django.db import transaction
-from django.utils.timezone import now, make_aware
 from django.contrib.auth.hashers import make_password
+from django.http import HttpResponse
+
+from django.utils.timezone import now, make_aware
+from django.utils.encoding import force_bytes
 
 from zoo.zoo_meta.models import EntitySchema, RelationSchema
 from zoo.zoo_data.models import Entity, Relation, Transaction, Log
@@ -141,6 +146,16 @@ def adresses(individu):
     return adresses
 
 
+def fratrie(individu):
+    assert individu.content['statut_legal'] == 'mineur'
+
+    def helper_fratrie():
+        for parent, relp in parents(individu):
+            for enfant, rele in enfants(parent):
+                yield enfant
+    return set(helper_fratrie())
+
+
 def adresses_norel(individu):
     return [adresse for adresse, rel in adresses(individu)]
 
@@ -1249,3 +1264,21 @@ def individu_caption(individu):
     if c['date_de_naissance']:
         s += u' - ' + c['date_de_naissance']
     return s
+
+
+def csv_export_response(rows, filename):
+    if sys.version >= (3,):
+        with io.StringIO(newline='') as f:
+            writer = csv.writer(f)
+            for row in rows:
+                writer.writerow(map(str, row))
+            r = HttpResponse(f.getvalue(), content_type='text/csv')
+    else:
+        with io.BytesIO() as f:
+            writer = csv.writer(f)
+            for row in rows:
+                writer.writerow(map(force_bytes, row))
+            r = HttpResponse(f.getvalue(), content_type='text/csv')
+    r['Content-Disposition'] = 'attachment; filename="%s"' % filename
+    return r
+

zoo/zoo_nanterre/views.py

@@ -17,19 +17,28 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import csv
+import itertools
+import time
 
 from django.template.response import TemplateResponse
 from django.views.generic import TemplateView
 from django.shortcuts import redirect, get_object_or_404
-from django.http import Http404, FileResponse
-from django.db.transaction import non_atomic_requests
+from django.http import Http404, FileResponse, HttpResponseRedirect
+from django.db.transaction import non_atomic_requests, atomic
 from django.db import connection
-from django.contrib.auth.decorators import permission_required
+from django.conf import settings
+from django.core.cache import cache
+from django.utils.timezone import now
 
-from zoo.zoo_data.models import Job
+from django.contrib.auth.decorators import permission_required
+from django.contrib import messages
+
+from zoo.zoo_data.models import Entity
 
 from . import forms
 from .synchronize_federations import SynchronizeFederationsAction
+from .inactivity import Inactivity
+from . import utils
 
 
 class Demo(TemplateView):
@@ -186,3 +195,84 @@ def synchronize_federations_apply(request, job_id, model_admin, **kwargs):
             raise Http404
         job.action.set_apply(job)
     return redirect('admin:synchronize-federations')
+
+
+def fiches_inactives():
+    inactivity = Inactivity(
+        child_delay=getattr(settings, 'ZOO_NANTERRE_INACTIVITY_CHILD_DELAY', 365),
+        adult_delay=getattr(settings, 'ZOO_NANTERRE_INACTIVITY_ADULT_DELAY', 365),
+    )
+    fiches = []
+    for child in itertools.chain(inactivity.deletable_children, inactivity.deletable_adults):
+        utils.PersonSearch.add_age(child)
+        fiches.append({
+            'id': child.id,
+            'prenoms': child.content['prenoms'],
+            'nom_de_naissance': child.content['nom_de_naissance'],
+            'nom_d_usage': child.content['nom_d_usage'],
+            'date_de_naissance': child.content['date_de_naissance'],
+            'statut_legal': child.content['statut_legal'],
+            'age': child.age_label,
+        })
+    fiches.sort(key=lambda f: f['id'])
+    return fiches
+
+
+@permission_required('zoo_data.action1_entity')
+@atomic
+def inactive_index(request, model_admin, *args, **kwargs):
+    try:
+        timestamp, fiches = cache.get('fiches-inactives')
+
+        if 'recompute' in request.GET:
+            cache.delete('fiches-inactives')
+            return HttpResponseRedirect(request.path)
+
+        # delete operation
+        if 'delete' in request.GET and request.method == 'POST':
+            new_timestamp = now()
+            new_fiches = fiches_inactives()
+            if fiches != new_fiches:
+                messages.warning(request, u'Les fiches à supprimer ont changées.')
+                cache.set('fiches-inactives', (new_timestamp, new_fiches), 365 * 24 * 3600)
+            else:
+                Entity.objects.filter(id__in=[fiche['id'] for fiche in fiches]).delete()
+                messages.info(request, u'%d fiches ont été supprimées.' % len(fiches))
+                cache.delete('fiches-inactives')
+            return HttpResponseRedirect(request.path)
+        duration = None
+        queries = None
+    except (TypeError, ValueError):
+        try:
+            connection.force_debug_cursor = True
+            start = time.time()
+            fiches = fiches_inactives()
+            queries = len(connection.queries_log)
+            duration = time.time() - start
+            timestamp = now()
+            cache.set('fiches-inactives', (timestamp, fiches), 365 * 24 * 3600)
+        finally:
+            connection.force_debug_cursor = False
+
+    # download csv export
+    if 'csv' in request.GET:
+        header = ['id', 'prenoms', 'nom_d_usage', 'nom_de_naissance',
+                  'date_de_naissance', 'statut_legal', 'age']
+
+        def rows():
+            yield header
+            for fiche in fiches:
+                yield [fiche[key] for key in header]
+        return utils.csv_export_response(rows(), 'fiches-inactives-%s.csv' % timestamp)
+
+    context = dict(
+        model_admin.admin_site.each_context(request),
+        title='Fiches inactives à supprimer',
+        fiches=fiches,
+        timestamp=timestamp,
+        duration=duration,
+        queries=queries,
+        child_delay=getattr(settings, 'ZOO_NANTERRE_INACTIVITY_CHILD_DELAY', 365),
+        adult_delay=getattr(settings, 'ZOO_NANTERRE_INACTIVITY_ADULT_DELAY', 365),
+    )
+    return TemplateResponse(request, "admin/zoo_data/entity/inactive_index.html", context)