import datetime from functools import wraps from django.contrib.auth.mixins import LoginRequiredMixin from django.conf import settings from django.core import serializers from django.core.exceptions import PermissionDenied from django.http import HttpResponse, HttpResponseRedirect from django.urls import reverse from django.views.decorators.csrf import csrf_exempt from django.views.generic import TemplateView from django.views.generic.list import ListView from logtracker.journal.models import Entry from logtracker.journal.journalstream import get_journal_entries class APIEntriesList(LoginRequiredMixin, ListView): def get_queryset(self): lines = self.request.GET.get('lines', 100) host = self.request.GET.get('host', '') return Entry.objects.dump(host=host, lines=lines) def get(self, request, *args, **kwargs): queryset = self.get_queryset() response = serializers.serialize("json", queryset) return HttpResponse(response, content_type='application/json') class EntriesList(LoginRequiredMixin, ListView): template_name = 'journal/entry_list.html' def get_queryset(self): lines = self.request.GET.get('lines', 100) host = self.request.GET.get('host') options = {k: v for k, v in self.request.GET.items() if k not in ('lines', 'host', 'timestamp')} return Entry.objects.dump(host=host, lines=lines, **options) def get_context_data(self, **kwargs): data = super().get_context_data(**kwargs) data['hosts'] = Entry.objects.hosts data['units'] = Entry.objects.units return data class HomeView(LoginRequiredMixin, TemplateView): template_name = 'home.html' def post(self, request, *args, **kwargs): url = '%s?%s' % (reverse('mail-api'), '%s=%s' % (request.POST['field'], request.POST['address'])) return HttpResponseRedirect(url) def get_chunks(gen, n): chunk = [] ended = False while True: for i in range(n): try: chunk.append(next(gen)) except StopIteration: ended = True yield chunk chunk = [] if ended: break def ssl_client_verify(view): @wraps(view) def wrapper(request, *args, **kwargs): headers = request.META if headers.get('HTTP_X_SSL') == "1" and ( headers.get('HTTP_X_SSL_CLIENT_VERIFY') == "0" or headers.get('HTTP_X_SSL_CLIENT_VERIFY') == "SUCCESS" ): cn = headers.get('HTTP_X_SSL_CLIENT_CN') dn = headers.get('HTTP_X_SSL_CLIENT_DN') if cn: request.host_verified = cn else: request.host_verified = dn.split(',')[0].split('=')[1] if settings.CA_ISSUER: ca_issuer = headers.get('HTTP_X_SSL_ISSUER') if ca_issuer != settings.CA_ISSUER: raise PermissionDenied else: if settings.DEBUG: request.host_verified = 'test_host' else: raise PermissionDenied return view(request, *args, **kwargs) return wrapper @ssl_client_verify @csrf_exempt def UploadView(request, debug=False): if request.method == "POST": count = 0 data = {} new_entries = [] now = datetime.datetime.now() timestamp = now journal_stream = request.META.get('wsgi.input') for chunk in get_chunks(get_journal_entries(journal_stream), 20): for el in chunk: data = {k: v for k, v in el} try: timestamp = datetime.datetime.fromtimestamp(int(data['__REALTIME_TIMESTAMP']) / 1000000) except (KeyError, TypeError, ValueError): continue if (now - timestamp).days > settings.JOURNAL_HISTORY: continue entry = Entry(timestamp=timestamp, host=request.host_verified, data=data) new_entries.append(entry) count += 1 if debug and count % 1000 == 0: print(count, timestamp) Entry.objects.bulk_create(new_entries) new_entries = [] if debug: elapsed = datetime.datetime.now() - now print('elapsed: %s' % elapsed) print('count: %s' % count) return HttpResponse('added %s' % count) raise PermissionDenied