cityweb: add expired files deletion (#76631)

passerelle/apps/cityweb/migrations/0005_cityweb_files_expiration.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.18 on 2023-04-28 08:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('cityweb', '0004_cityweb_remote_sftp'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='cityweb',
+            name='files_expiration',
+            field=models.IntegerField(
+                default=180,
+                verbose_name='Expiration time, in days, after which the generated files will be deleted',
+            ),
+        ),
+    ]

passerelle/apps/cityweb/models.py

@@ -15,8 +15,11 @@
 
 import json
 import os
+from datetime import datetime, timedelta
 
 from django.core.files.storage import default_storage
+from django.db import models
+from django.utils.timezone import make_aware, now
 from django.utils.translation import gettext_lazy as _
 
 from passerelle.base.models import BaseResource
@@ -38,6 +41,10 @@ from .cityweb import (
 class CityWeb(BaseResource):
     category = _('Civil Status Connectors')
     remote_sftp = SFTPField(verbose_name=_('Remote SFTP directory for outgoing files'), blank=True)
+    files_expiration = models.IntegerField(
+        verbose_name=_('Expiration time, in days, after which the generated files will be deleted'),
+        default=180,
+    )
 
     class Meta:
         verbose_name = "CityWeb - Demande d'acte d'état civil"
@@ -65,6 +72,14 @@ class CityWeb(BaseResource):
     def basepath(self):
         return os.path.join(default_storage.path('cityweb'), self.slug)
 
+    def daily(self):
+        super().daily()
+        for f in os.listdir(self.basepath):
+            demand_file = os.path.join(self.basepath, f)
+            file_mtime = datetime.fromtimestamp(os.path.getmtime(demand_file))
+            if make_aware(file_mtime) < now() - timedelta(days=self.files_expiration):
+                os.unlink(demand_file)
+
     @endpoint(perm='can_access', description=_('Get title list'))
     def titles(self, request):
         return {'data': TITLES}

tests/test_cityweb.py

@@ -21,6 +21,7 @@ import stat
 
 import pytest
 from django.core.files.storage import default_storage
+from django.utils import timezone
 from lxml import etree
 from lxml import objectify as xobject
 
@@ -284,3 +285,14 @@ def test_demand_creation_with_remote_sftp(app, setup, payload, sftpserver):
         expected_filename = '%s.xml' % resp.json['data']['demand_id']
         with sftpserver.serve_content({'input': {expected_filename: 'content'}}):
             setup.jobs()
+
+
+def test_expired_files_deletion(app, setup, payload, freezer):
+    for payload_value in payload.values():
+        resp = app.post_json('/cityweb/test/create', params=payload_value, status=200)
+        expected_filename = '%s.xml' % resp.json['data']['demand_id']
+        assert os.path.exists(default_storage.path(os.path.join('cityweb', 'test', expected_filename)))
+
+    freezer.move_to(timezone.now() + timezone.timedelta(days=190))
+    setup.daily()
+    assert os.listdir(default_storage.path(os.path.join('cityweb', 'test'))) == []