misc: order results starting with oldest entry by default, add pagination

debian/bin/logtracker

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 args=$*
-test $# -eq 0 && args=dump
+test $# -eq 0 && args=show
 
 if [ "$(whoami)" != "logtracker" ]; then
     sudo -H -u logtracker logtracker-manage $args

logtracker/journal/forms.py

@@ -1,14 +1,15 @@
 import datetime
-
 from django import forms
+from django.conf import settings
 from logtracker.journal.models import Entry
 
 
 class EntriesForm(forms.Form):
     host = forms.MultipleChoiceField(required=False)
     _systemd_unit = forms.MultipleChoiceField(required=False)
-    priority = forms.MultipleChoiceField(required=False, choices=[(i, i) for i in range(1, 7)])
-    since = forms.DateTimeField(required=False, initial=datetime.datetime.now())
+    priority = forms.ChoiceField(required=False, choices=reversed([(i, i) for i in range(1, 8)]))
+    since = forms.DateTimeField(required=False, initial=datetime.datetime.now() - datetime.timedelta(days=settings.LOGTRACKER_HISTORY))
+    tail = forms.BooleanField(required=False)
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

logtracker/journal/management/commands/clean_journal.py

@@ -10,7 +10,7 @@ class Command(BaseCommand):
     help = "Remove old entries"
 
     def add_arguments(self, parser):
-        parser.add_argument("--keep", type=int, default=settings.JOURNAL_HISTORY)
+        parser.add_argument("--keep", type=int, default=settings.LOGTRACKER_HISTORY)
 
     def handle(self, *args, **options):
         limit = datetime.datetime.now() - datetime.timedelta(days=options['keep'])

logtracker/journal/management/commands/show.py

@@ -1,8 +1,10 @@
 from django.core.management.base import BaseCommand, CommandError
 from logtracker.journal.models import Entry
+from pydoc import pager
 import os
 import textwrap
 
+
 def bold(txt):
     bold = "\033[1m"
     end = "\033[0m"
@@ -15,17 +17,25 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument("--full", action="store_true")
         parser.add_argument("--since")
-        parser.add_argument("-n", "--lines", default=100)
-        parser.add_argument('options', nargs="*")
+        parser.add_argument("-n", "--lines", help="tail n lines")
+        parser.add_argument("-p", "--priority")
+        parser.add_argument("--no-pager", action="store_true")
+        parser.add_argument('options', nargs="*", help="any systemd selector k=v")
 
     def handle(self, *args, **options):
+        if options['no_pager'] or options['lines']:
+            for l in self.get_lines(**options):
+                print(l)
+        else:
+            pager('\n'.join([l for l in self.get_lines(**options)]))
+
+    def get_lines(self, **options):
         _, columns = os.popen("stty size", "r").read().split()
         kwargs = {k: v for (k, v) in [o.split('=') for o in options['options']]}
-        for entry in Entry.objects.dump(lines=options['lines'], since=options['since'], **kwargs):
+        for entry in Entry.objects.extract(lines=options['lines'], since=options['since'], priority=options['priority'], **kwargs):
             line = "%s %s %s %s" % (
                 entry.timestamp.astimezone().strftime("%b %d %X"),
-                entry.host,
-                entry.unit,
+                entry.host, entry.unit,
                 entry.data.get("MESSAGE"),
             )
             priority = entry.data.get("PRIORITY")
@@ -33,4 +43,4 @@ class Command(BaseCommand):
                 line = textwrap.shorten(line, int(columns))
             if priority and int(priority) < 4:
                 line = bold(line)
-            print(line)
+            yield line

logtracker/journal/models.py

@@ -10,7 +10,7 @@ from django.utils import timezone
 
 class EntryManager(models.Manager):
 
-    def dump(self, since=None, host=None, lines=100, **kwargs):
+    def extract(self, since=None, host=None, lines=None, **kwargs):
         qs = Entry.objects.all()
         if since:
             timestamp = timezone.make_aware(datetime.datetime.strptime(since, "%Y-%m-%d %H:%M:%S"))
@@ -18,17 +18,21 @@ class EntryManager(models.Manager):
         if host:
             qs = qs.filter(host__in=host)
         qs = qs.filter(**self.parse_options(**kwargs))
-        qs = qs.order_by('timestamp')
         if lines:
-            qs = qs[:int(lines)]
+            qs = qs.order_by('-timestamp')
+            qs = qs[:int(lines)][::-1]
+        else:
+            qs = qs.order_by('timestamp')
         return qs
 
     def parse_options(self, **kwargs):
         # todo mode='contains' require jsonb KeyTextTransform and probably annotations
         options = {}
         for k, v in kwargs.items():
-            if k.upper() == 'csrfmiddlewaretoken':
+            if not v or v == ['']:
                 continue
+            if k == 'priority':
+                v = [str(n) for n in range(0, int(v))]
             options['data__' + k.upper() + '__in'] = v
         return options
 

logtracker/journal/templates/home.html

@@ -7,7 +7,7 @@
 <b>Journal entries</b>
 
 <ul>
-    <li><a href="{% url 'journal' %}?lines=100">Recent log entries</a> (<a href="{% url 'journal-api' %}?lines=100">api</a>)</li>
+    <li><a href="{% url 'journal' %}?lines=100">Recent log entries</a> (<a href="{% url 'journal-api' %}?tail=on">api</a>)</li>
 </ul>
 
 <b>Emails</b>

logtracker/journal/templates/journal/entry_list.html

@@ -1,7 +1,25 @@
 {% extends "base.html" %}
 
 {% block content %}
-<p><a href="{% url 'home' %}">Logtracker</a> - journal entries</p>
+<p><a href="{% url 'home' %}">Logtracker</a> - journal entries : {{ page_obj.paginator.count }}
+
+    <span class="step-links">
+        {% if page_obj.has_previous %}
+            <a href="?page=1">&laquo; first</a>
+            <a href="?page={{ page_obj.previous_page_number }}">previous</a>
+        {% endif %}
+
+        <span class="current">
+            Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.
+        </span>
+
+        {% if page_obj.has_next %}
+            <a href="?page={{ page_obj.next_page_number }}">next</a>
+            <a href="?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
+        {% endif %}
+    </span>
+
+</p>
 
 <form action="." style="display:inline;">
     {{ form.as_table }}
@@ -30,4 +48,6 @@
 {% endfor %}
     </tbody>
 </table>
+
+
 {% endblock %}

logtracker/journal/views.py

@@ -10,7 +10,8 @@ from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic.base import View, TemplateView
-from django.views.generic.edit import FormView
+from django.views.generic.list import ListView
+from django.views.generic.edit import FormMixin
 
 from logtracker.journal.models import Entry
 from logtracker.journal.journalstream import get_journal_entries
@@ -19,11 +20,16 @@ from logtracker.journal.forms import EntriesForm
 
 class APIEntriesList(LoginRequiredMixin, View):
     def get_queryset(self):
-        lines = self.request.GET.get('lines', 100)
+        tail = self.request.GET.get('tail')
+        if tail == "on":
+            lines = 20
+        else:
+            lines = None
         since = self.request.GET.get('since')
         host = self.request.GET.getlist('host')
-        options = {k: v for k, v in self.request.GET.lists() if k not in ('lines', 'host', 'since')}
-        return Entry.objects.dump(since=since, host=host, lines=lines, **options)
+        _systemd_unit = self.request.GET.getlist('_systemd_unit')
+        priority = self.request.GET.get('priority')
+        return Entry.objects.extract(since=since, host=host, lines=lines, _systemd_unit=_systemd_unit, priority=priority)
 
     def get(self, request, *args, **kwargs):
         queryset = self.get_queryset()
@@ -32,17 +38,24 @@ class APIEntriesList(LoginRequiredMixin, View):
 
 
 @method_decorator(csrf_exempt, name='dispatch')
-class EntriesList(LoginRequiredMixin, FormView):
+class EntriesList(LoginRequiredMixin, FormMixin, ListView):
     template_name = 'journal/entry_list.html'
     form_class = EntriesForm
+    paginate_by = 100
+
+    def get_queryset(self, **kwargs):
+        return APIEntriesList.get_queryset(self)
 
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
-        data['object_list'] = APIEntriesList.get_queryset(self)
+        data["count"] = len(data['object_list'])
         return data
 
     def get_initial(self):
-        return {k: v for k, v in self.request.GET.lists()}
+        initials = {k: v for k, v in self.request.GET.items()}
+        for x in ['host', '_systemd_unit']:
+            initials.update({x: self.request.GET.getlist(x)})
+        return initials
 
 
 class HomeView(LoginRequiredMixin, TemplateView):
@@ -108,7 +121,7 @@ def UploadView(request, debug=False):
             for el in chunk:
                 data = {k: v for k, v in el}
                 timestamp = datetime.datetime.fromtimestamp(int(data['__REALTIME_TIMESTAMP']) / 1000000)
-                if (now - timestamp).days > settings.JOURNAL_HISTORY:
+                if (now - timestamp).days > settings.LOGTRACKER_HISTORY:
                     continue
                 entry = Entry(timestamp=timestamp, host=request.host_verified, data=data)
                 new_entries.append(entry)

logtracker/settings.py

@@ -27,7 +27,7 @@ DEBUG = False
 ALLOWED_HOSTS = []
 
 CA_ISSUER = None  # dn of the trusted ca certificate; None means all (depending on the underlying haproxy/nginx configuration)
-JOURNAL_HISTORY = 7
+LOGTRACKER_HISTORY = 7
 MAIL_HISTORY = 7
 
 # Application definition

tests/settings.py

@@ -27,7 +27,7 @@ DEBUG = True
 
 ALLOWED_HOSTS = []
 
-JOURNAL_HISTORY = 1000
+LOGTRACKER_HISTORY = 1000
 MAIL_HISTORY = 1000
 
 # Application definition