From e7ba3d4b72d700373a54eac3996ac48262d05e67 Mon Sep 17 00:00:00 2001 From: Christophe Siraut Date: Fri, 30 Oct 2020 11:34:11 +0100 Subject: [PATCH] misc: order results starting with oldest entry by default, add pagination --- debian/bin/logtracker | 2 +- logtracker/journal/forms.py | 7 +++-- .../management/commands/clean_journal.py | 2 +- .../management/commands/{dump.py => show.py} | 22 ++++++++++---- logtracker/journal/models.py | 12 +++++--- logtracker/journal/templates/home.html | 2 +- .../journal/templates/journal/entry_list.html | 22 +++++++++++++- logtracker/journal/views.py | 29 ++++++++++++++----- logtracker/settings.py | 2 +- tests/settings.py | 2 +- 10 files changed, 75 insertions(+), 27 deletions(-) rename logtracker/journal/management/commands/{dump.py => show.py} (57%) diff --git a/debian/bin/logtracker b/debian/bin/logtracker index af8b38b..9c67ada 100755 --- a/debian/bin/logtracker +++ b/debian/bin/logtracker @@ -1,7 +1,7 @@ #!/bin/sh args=$* -test $# -eq 0 && args=dump +test $# -eq 0 && args=show if [ "$(whoami)" != "logtracker" ]; then sudo -H -u logtracker logtracker-manage $args diff --git a/logtracker/journal/forms.py b/logtracker/journal/forms.py index 429ffac..ecc32c1 100644 --- a/logtracker/journal/forms.py +++ b/logtracker/journal/forms.py @@ -1,14 +1,15 @@ import datetime - from django import forms +from django.conf import settings from logtracker.journal.models import Entry class EntriesForm(forms.Form): host = forms.MultipleChoiceField(required=False) _systemd_unit = forms.MultipleChoiceField(required=False) - priority = forms.MultipleChoiceField(required=False, choices=[(i, i) for i in range(1, 7)]) - since = forms.DateTimeField(required=False, initial=datetime.datetime.now()) + priority = forms.ChoiceField(required=False, choices=reversed([(i, i) for i in range(1, 8)])) + since = forms.DateTimeField(required=False, initial=datetime.datetime.now() - datetime.timedelta(days=settings.LOGTRACKER_HISTORY)) + tail = forms.BooleanField(required=False) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/logtracker/journal/management/commands/clean_journal.py b/logtracker/journal/management/commands/clean_journal.py index 8fa6659..ea4bcb7 100644 --- a/logtracker/journal/management/commands/clean_journal.py +++ b/logtracker/journal/management/commands/clean_journal.py @@ -10,7 +10,7 @@ class Command(BaseCommand): help = "Remove old entries" def add_arguments(self, parser): - parser.add_argument("--keep", type=int, default=settings.JOURNAL_HISTORY) + parser.add_argument("--keep", type=int, default=settings.LOGTRACKER_HISTORY) def handle(self, *args, **options): limit = datetime.datetime.now() - datetime.timedelta(days=options['keep']) diff --git a/logtracker/journal/management/commands/dump.py b/logtracker/journal/management/commands/show.py similarity index 57% rename from logtracker/journal/management/commands/dump.py rename to logtracker/journal/management/commands/show.py index 11089ac..deee6ce 100644 --- a/logtracker/journal/management/commands/dump.py +++ b/logtracker/journal/management/commands/show.py @@ -1,8 +1,10 @@ from django.core.management.base import BaseCommand, CommandError from logtracker.journal.models import Entry +from pydoc import pager import os import textwrap + def bold(txt): bold = "\033[1m" end = "\033[0m" @@ -15,17 +17,25 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("--full", action="store_true") parser.add_argument("--since") - parser.add_argument("-n", "--lines", default=100) - parser.add_argument('options', nargs="*") + parser.add_argument("-n", "--lines", help="tail n lines") + parser.add_argument("-p", "--priority") + parser.add_argument("--no-pager", action="store_true") + parser.add_argument('options', nargs="*", help="any systemd selector k=v") def handle(self, *args, **options): + if options['no_pager'] or options['lines']: + for l in self.get_lines(**options): + print(l) + else: + pager('\n'.join([l for l in self.get_lines(**options)])) + + def get_lines(self, **options): _, columns = os.popen("stty size", "r").read().split() kwargs = {k: v for (k, v) in [o.split('=') for o in options['options']]} - for entry in Entry.objects.dump(lines=options['lines'], since=options['since'], **kwargs): + for entry in Entry.objects.extract(lines=options['lines'], since=options['since'], priority=options['priority'], **kwargs): line = "%s %s %s %s" % ( entry.timestamp.astimezone().strftime("%b %d %X"), - entry.host, - entry.unit, + entry.host, entry.unit, entry.data.get("MESSAGE"), ) priority = entry.data.get("PRIORITY") @@ -33,4 +43,4 @@ class Command(BaseCommand): line = textwrap.shorten(line, int(columns)) if priority and int(priority) < 4: line = bold(line) - print(line) + yield line diff --git a/logtracker/journal/models.py b/logtracker/journal/models.py index 0f97049..9791738 100644 --- a/logtracker/journal/models.py +++ b/logtracker/journal/models.py @@ -10,7 +10,7 @@ from django.utils import timezone class EntryManager(models.Manager): - def dump(self, since=None, host=None, lines=100, **kwargs): + def extract(self, since=None, host=None, lines=None, **kwargs): qs = Entry.objects.all() if since: timestamp = timezone.make_aware(datetime.datetime.strptime(since, "%Y-%m-%d %H:%M:%S")) @@ -18,17 +18,21 @@ class EntryManager(models.Manager): if host: qs = qs.filter(host__in=host) qs = qs.filter(**self.parse_options(**kwargs)) - qs = qs.order_by('timestamp') if lines: - qs = qs[:int(lines)] + qs = qs.order_by('-timestamp') + qs = qs[:int(lines)][::-1] + else: + qs = qs.order_by('timestamp') return qs def parse_options(self, **kwargs): # todo mode='contains' require jsonb KeyTextTransform and probably annotations options = {} for k, v in kwargs.items(): - if k.upper() == 'csrfmiddlewaretoken': + if not v or v == ['']: continue + if k == 'priority': + v = [str(n) for n in range(0, int(v))] options['data__' + k.upper() + '__in'] = v return options diff --git a/logtracker/journal/templates/home.html b/logtracker/journal/templates/home.html index 6fd19b4..830d3c3 100644 --- a/logtracker/journal/templates/home.html +++ b/logtracker/journal/templates/home.html @@ -7,7 +7,7 @@ Journal entries Emails diff --git a/logtracker/journal/templates/journal/entry_list.html b/logtracker/journal/templates/journal/entry_list.html index 5a0ce78..532da13 100644 --- a/logtracker/journal/templates/journal/entry_list.html +++ b/logtracker/journal/templates/journal/entry_list.html @@ -1,7 +1,25 @@ {% extends "base.html" %} {% block content %} -

Logtracker - journal entries

+

Logtracker - journal entries : {{ page_obj.paginator.count }} + + + {% if page_obj.has_previous %} + « first + previous + {% endif %} + + + Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}. + + + {% if page_obj.has_next %} + next + last » + {% endif %} + + +

{{ form.as_table }} @@ -30,4 +48,6 @@ {% endfor %} + + {% endblock %} diff --git a/logtracker/journal/views.py b/logtracker/journal/views.py index a960f4f..eed162e 100644 --- a/logtracker/journal/views.py +++ b/logtracker/journal/views.py @@ -10,7 +10,8 @@ from django.urls import reverse from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt from django.views.generic.base import View, TemplateView -from django.views.generic.edit import FormView +from django.views.generic.list import ListView +from django.views.generic.edit import FormMixin from logtracker.journal.models import Entry from logtracker.journal.journalstream import get_journal_entries @@ -19,11 +20,16 @@ from logtracker.journal.forms import EntriesForm class APIEntriesList(LoginRequiredMixin, View): def get_queryset(self): - lines = self.request.GET.get('lines', 100) + tail = self.request.GET.get('tail') + if tail == "on": + lines = 20 + else: + lines = None since = self.request.GET.get('since') host = self.request.GET.getlist('host') - options = {k: v for k, v in self.request.GET.lists() if k not in ('lines', 'host', 'since')} - return Entry.objects.dump(since=since, host=host, lines=lines, **options) + _systemd_unit = self.request.GET.getlist('_systemd_unit') + priority = self.request.GET.get('priority') + return Entry.objects.extract(since=since, host=host, lines=lines, _systemd_unit=_systemd_unit, priority=priority) def get(self, request, *args, **kwargs): queryset = self.get_queryset() @@ -32,17 +38,24 @@ class APIEntriesList(LoginRequiredMixin, View): @method_decorator(csrf_exempt, name='dispatch') -class EntriesList(LoginRequiredMixin, FormView): +class EntriesList(LoginRequiredMixin, FormMixin, ListView): template_name = 'journal/entry_list.html' form_class = EntriesForm + paginate_by = 100 + + def get_queryset(self, **kwargs): + return APIEntriesList.get_queryset(self) def get_context_data(self, **kwargs): data = super().get_context_data(**kwargs) - data['object_list'] = APIEntriesList.get_queryset(self) + data["count"] = len(data['object_list']) return data def get_initial(self): - return {k: v for k, v in self.request.GET.lists()} + initials = {k: v for k, v in self.request.GET.items()} + for x in ['host', '_systemd_unit']: + initials.update({x: self.request.GET.getlist(x)}) + return initials class HomeView(LoginRequiredMixin, TemplateView): @@ -108,7 +121,7 @@ def UploadView(request, debug=False): for el in chunk: data = {k: v for k, v in el} timestamp = datetime.datetime.fromtimestamp(int(data['__REALTIME_TIMESTAMP']) / 1000000) - if (now - timestamp).days > settings.JOURNAL_HISTORY: + if (now - timestamp).days > settings.LOGTRACKER_HISTORY: continue entry = Entry(timestamp=timestamp, host=request.host_verified, data=data) new_entries.append(entry) diff --git a/logtracker/settings.py b/logtracker/settings.py index 4892200..2ae9173 100644 --- a/logtracker/settings.py +++ b/logtracker/settings.py @@ -27,7 +27,7 @@ DEBUG = False ALLOWED_HOSTS = [] CA_ISSUER = None # dn of the trusted ca certificate; None means all (depending on the underlying haproxy/nginx configuration) -JOURNAL_HISTORY = 7 +LOGTRACKER_HISTORY = 7 MAIL_HISTORY = 7 # Application definition diff --git a/tests/settings.py b/tests/settings.py index f09bea3..a7b1cae 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -27,7 +27,7 @@ DEBUG = True ALLOWED_HOSTS = [] -JOURNAL_HISTORY = 1000 +LOGTRACKER_HISTORY = 1000 MAIL_HISTORY = 1000 # Application definition