misc: add time sorting

logtracker/journal/forms.py

@@ -0,0 +1,16 @@
+import datetime
+
+from django import forms
+from logtracker.journal.models import Entry
+
+
+class EntriesForm(forms.Form):
+    host = forms.MultipleChoiceField(required=False)
+    _systemd_unit = forms.MultipleChoiceField(required=False)
+    priority = forms.MultipleChoiceField(required=False, choices=[(i, i) for i in range(1, 7)])
+    since = forms.DateTimeField(required=False, initial=datetime.datetime.now())
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields['host'].choices = set([(e, e) for e in Entry.objects.hosts()])
+        self.fields['_systemd_unit'].choices = [(e, e) for e in Entry.objects.units()]

logtracker/journal/management/commands/dump.py

@@ -3,7 +3,6 @@ from logtracker.journal.models import Entry
 import os
 import textwrap
 
-
 def bold(txt):
     bold = "\033[1m"
     end = "\033[0m"
@@ -15,13 +14,14 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument("--full", action="store_true")
+        parser.add_argument("--since")
         parser.add_argument("-n", "--lines", default=100)
         parser.add_argument('options', nargs="*")
 
     def handle(self, *args, **options):
         _, columns = os.popen("stty size", "r").read().split()
         kwargs = {k: v for (k, v) in [o.split('=') for o in options['options']]}
-        for entry in Entry.objects.dump(lines=options['lines'], **kwargs):
+        for entry in Entry.objects.dump(lines=options['lines'], since=options['since'], **kwargs):
             line = "%s %s %s %s" % (
                 entry.timestamp.astimezone().strftime("%b %d %X"),
                 entry.host,

logtracker/journal/models.py

@@ -1,20 +1,26 @@
+import datetime
 import random
 
 from django.db import models
 from django.contrib.postgres.fields import JSONField
 from django.contrib.postgres.fields.jsonb import KeyTextTransform
 from django.core.cache import cache
+from django.utils import timezone
 
 
 class EntryManager(models.Manager):
 
-    def dump(self, timestamp=None, host=None, lines=100, **kwargs):
+    def dump(self, since=None, host=None, lines=100, **kwargs):
         qs = Entry.objects.all()
+        if since:
+            timestamp = timezone.make_aware(datetime.datetime.strptime(since, "%Y-%m-%d %H:%M:%S"))
+            qs = qs.filter(timestamp__gt=timestamp)
         if host:
             qs = qs.filter(host__in=host)
         qs = qs.filter(**self.parse_options(**kwargs))
+        qs = qs.order_by('timestamp')
         if lines:
-            qs = qs.order_by('-timestamp')[:int(lines)][::-1]
+            qs = qs[:int(lines)]
         return qs
 
     def parse_options(self, **kwargs):
@@ -29,20 +35,20 @@ class EntryManager(models.Manager):
     def hosts(self):
         value = cache.get('hosts')
         if value is None:
-            value = self.model.objects.all().values_list('host', flat=True).distinct()
+            value = sorted(self.model.objects.all().values_list('host', flat=True).distinct(), key=lambda x: x.lower())
             cache.set('hosts', value, 600 * random.random())
         return value
 
     def units(self):
         value = cache.get('units')
         if value is None:
-            value = self.model.objects.all().annotate(unit=KeyTextTransform('_SYSTEMD_UNIT', 'data')).values_list('unit', flat=True).distinct()
+            value = sorted([v for v in self.model.objects.all().annotate(unit=KeyTextTransform('_SYSTEMD_UNIT', 'data')).values_list('unit', flat=True).distinct() if v], key=lambda x: x.lower())
             cache.set('units', value, 600 * random.random())
         return value
 
 
 class Entry(models.Model):
-    timestamp = models.DateTimeField(auto_now_add=True, db_index=True)
+    timestamp = models.DateTimeField(db_index=True)
     host = models.CharField(max_length=128, db_index=True)
     data = JSONField()
     objects = EntryManager()

logtracker/journal/views.py

@@ -1,7 +1,6 @@
 import datetime
 from functools import wraps
 
-from django import forms
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.conf import settings
 from django.core import serializers
@@ -15,14 +14,16 @@ from django.views.generic.edit import FormView
 
 from logtracker.journal.models import Entry
 from logtracker.journal.journalstream import get_journal_entries
+from logtracker.journal.forms import EntriesForm
 
 
 class APIEntriesList(LoginRequiredMixin, View):
     def get_queryset(self):
         lines = self.request.GET.get('lines', 100)
+        since = self.request.GET.get('since')
         host = self.request.GET.getlist('host')
-        options = {k: v for k, v in self.request.GET.lists() if k not in ('lines', 'host', 'timestamp')}
-        return Entry.objects.dump(host=host, lines=lines, **options)
+        options = {k: v for k, v in self.request.GET.lists() if k not in ('lines', 'host', 'since')}
+        return Entry.objects.dump(since=since, host=host, lines=lines, **options)
 
     def get(self, request, *args, **kwargs):
         queryset = self.get_queryset()
@@ -30,12 +31,6 @@ class APIEntriesList(LoginRequiredMixin, View):
         return HttpResponse(response, content_type='application/json')
 
 
-class EntriesForm(forms.Form):
-    host = forms.MultipleChoiceField(required=False, choices=[(e, e) for e in Entry.objects.hosts()])
-    _systemd_unit = forms.MultipleChoiceField(required=False, choices=((u, u) for u in Entry.objects.units()))
-    priority = forms.MultipleChoiceField(required=False, choices=[(i, i) for i in range(1, 7)])
-
-
 @method_decorator(csrf_exempt, name='dispatch')
 class EntriesList(LoginRequiredMixin, FormView):
     template_name = 'journal/entry_list.html'
@@ -108,15 +103,11 @@ def UploadView(request, debug=False):
         data = {}
         new_entries = []
         now = datetime.datetime.now()
-        timestamp = now
         journal_stream = request.META.get('wsgi.input')
         for chunk in get_chunks(get_journal_entries(journal_stream), 20):
             for el in chunk:
                 data = {k: v for k, v in el}
-                try:
-                    timestamp = datetime.datetime.fromtimestamp(int(data['__REALTIME_TIMESTAMP']) / 1000000)
-                except (KeyError, TypeError, ValueError):
-                    continue
+                timestamp = datetime.datetime.fromtimestamp(int(data['__REALTIME_TIMESTAMP']) / 1000000)
                 if (now - timestamp).days > settings.JOURNAL_HISTORY:
                     continue
                 entry = Entry(timestamp=timestamp, host=request.host_verified, data=data)