misc: add settings JOURNAL_HISTORY and MAIL_HISTORY, adapt management commands and stream import functions

logtracker/journal/management/commands/clean_journal.py

@@ -1,16 +1,17 @@
-from django.core.management.base import BaseCommand, CommandError
-from django.db import connection
+import datetime
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from logtracker.journal.models import Entry
 
 
 class Command(BaseCommand):
     help = "Remove old entries"
 
     def add_arguments(self, parser):
-        parser.add_argument("--keep", type=int, default="100000")
+        parser.add_argument("--keep", type=int, default=settings.JOURNAL_HISTORY)
 
     def handle(self, *args, **options):
-        with connection.cursor() as cursor:
-            cursor.execute(
-                "DELETE FROM journal_entry WHERE id not in (SELECT id FROM journal_entry ORDER BY timestamp desc LIMIT %s)",
-                [options['keep']],
-            )
+        limit = datetime.datetime.now() - datetime.timedelta(days=options['keep'])
+        Entry.objects.filter(timestamp__lt=limit).delete()

logtracker/journal/views.py

@@ -77,8 +77,8 @@ def UploadView(request, debug=False):
         count = 0
         data = {}
         new_entries = []
-        start_timestamp = datetime.datetime.now()
-        timestamp = start_timestamp
+        now = datetime.datetime.now()
+        timestamp = now
         journal_stream = request.META.get('wsgi.input')
         for chunk in get_chunks(get_journal_entries(journal_stream), 20):
             for el in chunk:
@@ -87,6 +87,8 @@ def UploadView(request, debug=False):
                     timestamp = datetime.datetime.fromtimestamp(data['__REALTIME_TIMESTAMP'] / 1000000)
                 except (KeyError, TypeError, ValueError):
                     continue
+                if (now - timestamp).days > settings.JOURNAL_HISTORY:
+                    continue
                 entry = Entry(timestamp=timestamp, host=request.host_verified, data=data)
                 new_entries.append(entry)
                 count += 1

logtracker/mail/management/commands/clean_mail.py

@@ -1,16 +1,17 @@
-from django.core.management.base import BaseCommand, CommandError
-from django.db import connection
+import datetime
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from logtracker.mail.models import Mail
 
 
 class Command(BaseCommand):
-    help = "Remove old entries"
+    help = "Remove old mail entries"
 
     def add_arguments(self, parser):
-        parser.add_argument("--keep", type=int, default="100000")
+        parser.add_argument("--keep", type=int, default=settings.MAIL_HISTORY)
 
     def handle(self, *args, **options):
-        with connection.cursor() as cursor:
-            cursor.execute(
-                "DELETE FROM mail_mail WHERE id not in (SELECT id FROM journal_entry ORDER BY timestamp desc LIMIT %s)",
-                [options['keep']],
-            )
+        limit = datetime.datetime.now() - datetime.timedelta(days=options['keep'])
+        Mail.objects.filter(timestamp__lt=limit).delete()

logtracker/settings.py

@@ -26,6 +26,8 @@ DEBUG = False
 
 ALLOWED_HOSTS = []
 
+JOURNAL_HISTORY = 7
+MAIL_HISTORY = 7
 
 # Application definition
 

tests/settings.py

@@ -27,6 +27,8 @@ DEBUG = True
 
 ALLOWED_HOSTS = []
 
+JOURNAL_HISTORY = 1000
+MAIL_HISTORY = 1000
 
 # Application definition