add a fargo-cleanup command (#22682)

debian/fargo.cron.hourly

@@ -1,3 +1,4 @@
 #!/bin/sh
 
 sudo -u fargo /usr/bin/fargo-manage tenant_command clearsessions --all
+sudo -u fargo /usr/bin/fargo-manage tenant_command fargo-cleanup --all

fargo/fargo/management/commands/fargo-cleanup.py

@@ -0,0 +1,11 @@
+from django.core.management.base import NoArgsCommand
+
+from fargo.fargo.utils import cleanup
+
+
+class Command(NoArgsCommand):
+    help = 'Clean expired models'
+
+    def handle_noargs(self, **options):
+        cleanup()
+

fargo/fargo/managers.py

@@ -1,14 +1,21 @@
+import datetime
+
 from django.db import models
+from django.utils.timezone import now
 
 from . import utils
 
 
 class DocumentManager(models.Manager):
-    def clean(self):
-        '''Remove all documents not linked to an user'''
-        qs = self.filter(user_documents__isnull=True)
+    def cleanup(self, n=None):
+        '''Delete all orphaned documents'''
+        n = n or now()
+        # use a window of 60 seconds to be sure this document will never be used
+        qs = self.filter(creation_date__lt=n - datetime.timedelta(seconds=60))
+        qs = qs.filter(user_documents__isnull=True,
+                       oauth2_tempfiles__isnull=True)
         for document in qs:
-            qs.content.delete(False)
+            document.content.delete(False)
         qs.delete()
 
     def get_by_file(self, f):

fargo/fargo/migrations/0015_document_creation_date.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2018-03-23 15:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fargo', '0014_auto_20171016_0854'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='document',
+            name='creation_date',
+            field=models.DateTimeField(auto_now_add=True),
+        ),
+    ]

fargo/fargo/models.py

@@ -96,6 +96,7 @@ class UserDocument(models.Model):
                 self.document.mime_type.split('/')[0],
                 re.sub('[/\.+-]', '-', self.document.mime_type))
 
+
 class Validation(models.Model):
     '''Validation of a document as special kind for an user,
        the data field contains metadata extracted from the document.
@@ -169,6 +170,7 @@ class Document(models.Model):
     mime_type = models.CharField(
         max_length=256,
         blank=True)
+    creation_date = models.DateTimeField(auto_now_add=True)
 
     objects = managers.DocumentManager()
 
@@ -222,6 +224,7 @@ def create_thumbnail(sender, instance, created, **kwargs):
                          instance.thumbnail_full_path])
     threading.Thread(target=do).start()
 
+
 @receiver(post_delete, sender=Document)
 def delete_file(sender, instance, **kwargs):
     if instance.content:

fargo/fargo/utils.py

@@ -2,6 +2,7 @@ import hashlib
 
 from django.utils.timezone import utc
 from django.utils.encoding import smart_bytes
+from django.db import models
 
 try:
     import magic
@@ -40,3 +41,14 @@ def get_mime_type(path):
         if mime_type.startswith('cannot open'):
             mime_type = None
     return mime_type
+
+def cleanup_model(model, n=None):
+    manager = getattr(model, 'objects', None)
+    if hasattr(manager, 'cleanup'):
+        manager.cleanup(n=n)
+
+
+def cleanup(n=None):
+    for app in models.get_apps():
+        for model in models.get_models(app):
+            cleanup_model(model, n=n)

fargo/oauth2/migrations/0001_initial.py

@@ -37,7 +37,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('hash_key', models.CharField(max_length=128, serialize=False, primary_key=True)),
                 ('filename', models.CharField(max_length=512)),
-                ('document', models.ForeignKey(to='fargo.Document')),
+                ('document', models.ForeignKey(to='fargo.Document', related_name='oauth2_tempfiles')),
             ],
         ),
     ]

fargo/oauth2/models.py

@@ -15,11 +15,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import uuid
+import datetime
 
+from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.validators import URLValidator
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
+from django.db.models.query import QuerySet
+from django.utils.timezone import now
 
 from fargo.fargo.models import Document, UserDocument
 
@@ -63,6 +67,13 @@ class OAuth2Client(models.Model):
         return self.client_name
 
 
+class CleanupQuerySet(QuerySet):
+    def cleanup(self, n=None):
+        n = n or now()
+        threshold = n - datetime.timedelta(seconds=2 * self.model.get_lifetime())
+        self.filter(creation_date__lt=threshold).delete()
+
+
 class OAuth2Authorize(models.Model):
     client = models.ForeignKey(OAuth2Client)
     user_document = models.ForeignKey(UserDocument)
@@ -70,6 +81,14 @@ class OAuth2Authorize(models.Model):
     code = models.CharField(max_length=255, default=generate_uuid)
     creation_date = models.DateTimeField(auto_now_add=True)
 
+    objects = CleanupQuerySet.as_manager()
+
+    @classmethod
+    def get_lifetime(cls):
+        return max(
+            settings.FARGO_CODE_LIFETIME,
+            settings.FARGO_ACCESS_TOKEN_LIFETIME)
+
     def __repr__(self):
         return 'OAuth2Authorize for document %r' % self.user_document
 
@@ -77,6 +96,12 @@ class OAuth2Authorize(models.Model):
 class OAuth2TempFile(models.Model):
     uuid = models.CharField(max_length=32, default=generate_uuid, primary_key=True)
     client = models.ForeignKey(OAuth2Client)
-    document = models.ForeignKey(Document)
+    document = models.ForeignKey(Document, related_name='oauth2_tempfiles')
     filename = models.CharField(max_length=512)
     creation_date = models.DateTimeField(auto_now_add=True)
+
+    objects = CleanupQuerySet.as_manager()
+
+    @classmethod
+    def get_lifetime(cls):
+        return settings.FARGO_OAUTH2_TEMPFILE_LIFETIME

fargo/settings.py

@@ -228,6 +228,7 @@ INCLUDE_EDIT_LINK = False
 
 FARGO_CODE_LIFETIME = 300
 FARGO_ACCESS_TOKEN_LIFETIME = 3600
+FARGO_OAUTH2_TEMPFILE_LIFETIME = 86400
 
 local_settings_file = os.environ.get('FARGO_SETTINGS_FILE',
                                      os.path.join(

tests/test_commands.py

@@ -0,0 +1,54 @@
+import datetime
+
+from django.core.management import call_command
+
+from django.contrib.auth.models import User
+from fargo.fargo.models import UserDocument, Document
+from fargo.oauth2.models import OAuth2TempFile, OAuth2Client
+from django.core.files.base import ContentFile
+
+
+def test_cleanup(freezer, john_doe):
+    start = freezer()
+
+    client = OAuth2Client.objects.create(client_name='c', redirect_uris='')
+
+    foo = Document.objects.create(content=ContentFile('foo', name='foo.txt'))
+    bar = Document.objects.create(content=ContentFile('bar', name='bar.txt'))
+    UserDocument.objects.create(user=john_doe,
+                                document=foo,
+                                filename='foo.txt',
+                                title='',
+                                description='')
+    OAuth2TempFile.objects.create(document=bar, client=client, filename='bar.txt')
+
+    call_command('fargo-cleanup')
+
+    assert UserDocument.objects.all().count()
+    assert OAuth2TempFile.objects.all().count()
+    assert Document.objects.all().count() == 2
+
+    User.objects.all().delete()
+
+    assert not UserDocument.objects.all().count()
+    assert Document.objects.all().count() == 2
+
+    call_command('fargo-cleanup')
+
+    assert Document.objects.all().count() == 2
+
+    freezer.move_to(start + datetime.timedelta(seconds=120))
+    call_command('fargo-cleanup')
+
+    assert Document.objects.all().count() == 1
+
+    freezer.move_to(start + datetime.timedelta(days=3))
+
+    call_command('fargo-cleanup')
+
+    assert not OAuth2TempFile.objects.count()
+    assert Document.objects.count()
+
+    call_command('fargo-cleanup')
+
+    assert not Document.objects.count()

tox.ini

@@ -7,25 +7,26 @@ usedevelop = True
 whitelist_externals =
   /bin/mv
 setenv =
-	sqlite: DB_ENGINE=sqlite3
-	pg: DB_ENGINE=postgresql_psycopg2
-	DJANGO_SETTINGS_MODULE=fargo.settings
-	FARGO_SETTINGS_FILE=tests/settings.py
-	coverage: COVERAGE=--cov=fargo --cov-report xml
+        sqlite: DB_ENGINE=sqlite3
+        pg: DB_ENGINE=postgresql_psycopg2
+        DJANGO_SETTINGS_MODULE=fargo.settings
+        FARGO_SETTINGS_FILE=tests/settings.py
+        coverage: COVERAGE=--cov=fargo --cov-report xml
 deps =
-	dj18: django>=1.8,<1.9
-	dj18: django-tables2<1.1
-	dj111: django>=1.11,<1.12
-	dj111: django-tables2>=1.5
-	pytest>=3.3.0
-	pytest-cov
-	pytest-random
-	pytest-mock
-	pytest-django
-	django-webtest
-	WebTest
-	djangorestframework>=3.3,<3.4
+        dj18: django>=1.8,<1.9
+        dj18: django-tables2<1.1
+        dj111: django>=1.11,<1.12
+        dj111: django-tables2>=1.5
+        pytest>=3.3.0
+        pytest-cov
+        pytest-random
+        pytest-mock
+        pytest-django
+        pytest-freezegun
+        django-webtest
+        WebTest
+        djangorestframework>=3.3,<3.4
 commands =
-	py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests}
-	coverage: mv coverage.xml coverage-{envname}.xml
+        py.test {env:COVERAGE:} {posargs:--random --junit-xml=junit-{envname}.xml tests}
+        coverage: mv coverage.xml coverage-{envname}.xml