From 7297f1e8ef2e7923dfa9433ff73309e97b6d8a5f Mon Sep 17 00:00:00 2001 From: Christophe Siraut Date: Fri, 8 Nov 2019 15:19:22 +0100 Subject: [PATCH] start logtracker --- .gitignore | 19 +++ Jenkinsfile | 35 +++++ MANIFEST.in | 6 + README | 4 + debian/changelog | 5 + debian/compat | 1 + debian/control | 41 ++++++ debian/copyright | 38 +++++ debian/logtracker-agent-generator | 36 +++++ debian/logtracker-agent.README.Debian | 26 ++++ debian/logtracker-agent.dirs | 1 + debian/logtracker-agent.install | 1 + debian/logtracker-agent.service | 15 ++ debian/logtracker-agent@.service | 19 +++ debian/logtracker-manage | 22 +++ debian/logtracker.dirs | 4 + debian/logtracker.install | 4 + debian/logtracker.postinst | 48 +++++++ debian/logtracker.service | 21 +++ debian/nginx/logtracker-example.conf | 48 +++++++ debian/python3-logtracker.dirs | 1 + debian/rules | 14 ++ debian/settings.py | 37 +++++ debian/source/format | 1 + debian/uwsgi.ini | 7 + logtracker/__init__.py | 0 logtracker/agent/__init__.py | 0 logtracker/agent/agent.py | 44 ++++++ logtracker/agent/exim.py | 42 ++++++ logtracker/agent/journald.py | 30 ++++ logtracker/collection/__init__.py | 0 logtracker/collection/admin.py | 3 + .../collection/migrations/0001_initial.py | 31 ++++ logtracker/collection/migrations/__init__.py | 0 logtracker/collection/models.py | 16 +++ logtracker/collection/templates/base.html | 12 ++ .../collection/templates/entries/home.html | 18 +++ .../templates/entries/sender_list.html | 29 ++++ logtracker/collection/views.py | 28 ++++ logtracker/mail/__init__.py | 0 logtracker/mail/admin.py | 3 + logtracker/mail/apps.py | 5 + logtracker/mail/management/__init__.py | 0 .../mail/management/commands/__init__.py | 0 .../mail/management/commands/cleanup.py | 18 +++ .../mail/management/commands/collect.py | 25 ++++ logtracker/mail/management/commands/errors.py | 9 ++ .../mail/management/commands/get_entries.py | 12 ++ .../mail/management/commands/senders_count.py | 9 ++ logtracker/mail/models.py | 84 +++++++++++ logtracker/mail/templates/base.html | 12 ++ logtracker/mail/templates/entries/home.html | 18 +++ .../mail/templates/entries/sender_list.html | 29 ++++ logtracker/mail/views.py | 48 +++++++ logtracker/settings.py | 127 +++++++++++++++++ logtracker/urls.py | 18 +++ logtracker/utils.py | 15 ++ logtracker/wsgi.py | 16 +++ manage.py | 22 +++ setup.py | 134 ++++++++++++++++++ tests/__init__.py | 3 + tests/conftest.py | 48 +++++++ tests/settings.py | 123 ++++++++++++++++ tests/test_agent_journald.py | 5 + tests/test_base.py | 20 +++ tests/test_mail.py | 17 +++ tests/testdata.exim | 27 ++++ tests/testdata.journald | 10 ++ tox.ini | 28 ++++ 69 files changed, 1592 insertions(+) create mode 100644 .gitignore create mode 100644 Jenkinsfile create mode 100644 MANIFEST.in create mode 100644 README create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100755 debian/logtracker-agent-generator create mode 100644 debian/logtracker-agent.README.Debian create mode 100644 debian/logtracker-agent.dirs create mode 100644 debian/logtracker-agent.install create mode 100644 debian/logtracker-agent.service create mode 100644 debian/logtracker-agent@.service create mode 100755 debian/logtracker-manage create mode 100644 debian/logtracker.dirs create mode 100644 debian/logtracker.install create mode 100644 debian/logtracker.postinst create mode 100644 debian/logtracker.service create mode 100644 debian/nginx/logtracker-example.conf create mode 100644 debian/python3-logtracker.dirs create mode 100755 debian/rules create mode 100644 debian/settings.py create mode 100644 debian/source/format create mode 100644 debian/uwsgi.ini create mode 100644 logtracker/__init__.py create mode 100644 logtracker/agent/__init__.py create mode 100644 logtracker/agent/agent.py create mode 100755 logtracker/agent/exim.py create mode 100755 logtracker/agent/journald.py create mode 100644 logtracker/collection/__init__.py create mode 100644 logtracker/collection/admin.py create mode 100644 logtracker/collection/migrations/0001_initial.py create mode 100644 logtracker/collection/migrations/__init__.py create mode 100644 logtracker/collection/models.py create mode 100644 logtracker/collection/templates/base.html create mode 100644 logtracker/collection/templates/entries/home.html create mode 100644 logtracker/collection/templates/entries/sender_list.html create mode 100644 logtracker/collection/views.py create mode 100644 logtracker/mail/__init__.py create mode 100644 logtracker/mail/admin.py create mode 100644 logtracker/mail/apps.py create mode 100644 logtracker/mail/management/__init__.py create mode 100644 logtracker/mail/management/commands/__init__.py create mode 100644 logtracker/mail/management/commands/cleanup.py create mode 100644 logtracker/mail/management/commands/collect.py create mode 100644 logtracker/mail/management/commands/errors.py create mode 100644 logtracker/mail/management/commands/get_entries.py create mode 100644 logtracker/mail/management/commands/senders_count.py create mode 100644 logtracker/mail/models.py create mode 100644 logtracker/mail/templates/base.html create mode 100644 logtracker/mail/templates/entries/home.html create mode 100644 logtracker/mail/templates/entries/sender_list.html create mode 100644 logtracker/mail/views.py create mode 100644 logtracker/settings.py create mode 100644 logtracker/urls.py create mode 100644 logtracker/utils.py create mode 100644 logtracker/wsgi.py create mode 100755 manage.py create mode 100755 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/settings.py create mode 100644 tests/test_agent_journald.py create mode 100644 tests/test_base.py create mode 100644 tests/test_mail.py create mode 100644 tests/testdata.exim create mode 100644 tests/testdata.journald create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4227cbc --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +venv +*.ex +__pycache__ +*.egg-info +*.sqlite3 +*.offset +saml.* +.pybuild +build +debian/files +debian/logtracker +debian/python3-logtracker +debian/*.debhelper +debian/*.substvars +debian/debhelper-build-stamp +.coverage +coverage.xml +htmlcov/ +test_dj111_results.xml diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..20f4a5f --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,35 @@ +@Library('eo-jenkins-lib@master') import eo.Utils + +pipeline { + agent any + stages { + stage('Unit Tests') { + steps { + sh 'tox -rv' + } + } + stage('Packaging') { + steps { + script { + if (env.JOB_NAME == 'logtracker' && env.GIT_BRANCH == 'origin/master') { + sh 'sudo -H -u eobuilder /usr/local/bin/eobuilder -d buster logtracker' + } + } + } + } + } + post { + always { + script { + utils = new Utils() + utils.mail_notify(currentBuild, env, 'admin+logtracker@entrouvert.com') + utils.publish_coverage('coverage.xml') + utils.publish_coverage_native('index.html') + } + junit '*_results.xml' + } + success { + cleanWs() + } + } +} diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..bec7d24 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +recursive-include logtracker/collection/templates *.html +recursive-include logtracker/mail/templates *.html +include Jenkinsfile +include tests/testdata.exim +include tests/testdata.journald +include tox.ini diff --git a/README b/README new file mode 100644 index 0000000..25c098e --- /dev/null +++ b/README @@ -0,0 +1,4 @@ +Logtracker +=========== + +Logtracker is a django application to collect, aggregate and display log entries diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..5f3067c --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +logtracker (0.1-1) unstable; urgency=medium + + * Initial release (Closes: #nnnn) + + -- Christophe Siraut Thu, 07 Nov 2019 13:18:01 +0100 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +10 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..80fd227 --- /dev/null +++ b/debian/control @@ -0,0 +1,41 @@ +Source: logtracker +Section: unknown +Priority: optional +Maintainer: Christophe Siraut +Build-Depends: debhelper (>= 10), dh-python, python3-all, python3-setuptools, + python3-django, + python3-psycopg2, + python3-tz, + python3-pytest, + python3-pytest-django +Standards-Version: 4.1.3 +Homepage: https://git.entrouvert.org +X-Python3-Version: >= 3.2 +#Vcs-Browser: https://salsa.debian.org/debian/logtracker +#Vcs-Git: https://salsa.debian.org/debian/logtracker.git +#Testsuite: autopkgtest-pkg-python + +Package: python3-logtracker +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-psycopg2, + python3-tz +Description: Logtracker is a django application to expose email logs + This package installs the library for Python 3. + +Package: logtracker +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + postgresql, + python3-logtracker, + python3-django, + uwsgi, + uwsgi-plugin-python3 +Description: Logtracker service +Recommends: nginx + +Package: logtracker-agent +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends}, + python3-logtracker, +Description: Logtracker journald agent diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..2ecb899 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,38 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: logtracker +Source: + +Files: * +Copyright: + +License: + + + . + + +# If you want to use GPL v2 or later for the /debian/* files use +# the following clauses, or change it to suit. Delete these two lines +Files: debian/* +Copyright: 2019 Christophe Siraut +License: GPL-2+ + This package is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see + . + On Debian systems, the complete text of the GNU General + Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". + +# Please also look if there are files or directories which have a +# different copyright/license attached and list them here. +# Please avoid picking licenses with terms that are more restrictive than the +# packaged work, as it may make Debian's contributions unacceptable upstream. diff --git a/debian/logtracker-agent-generator b/debian/logtracker-agent-generator new file mode 100755 index 0000000..0ee296b --- /dev/null +++ b/debian/logtracker-agent-generator @@ -0,0 +1,36 @@ +#!/bin/sh + +# This systemd generator creates dependency symlinks that make all logtracker +# agents listed in /etc/logtracker/agents be started/stopped/reloaded +# when logtracker-agent.service is started/stopped/reloaded. + +set -eu + +GENDIR="$1" +WANTDIR="$1/logtracker-agent.service.wants" +SERVICEFILE="/lib/systemd/system/logtracker-agent@.service" +AUTOSTART="all" +CONFIG_DIR=/etc/logtracker/agents + +mkdir -p "$WANTDIR" + +# No agent automatically started +if test "x$AUTOSTART" = "xnone" ; then + exit 0 +fi + +if test "x$AUTOSTART" = "xall" -o -z "$AUTOSTART" ; then + for CONFIG in `cd $CONFIG_DIR; ls *.conf 2> /dev/null`; do + NAME=${CONFIG%%.conf} + ln -s "$SERVICEFILE" "$WANTDIR/logtracker-agent@$NAME.service" + done +else + for NAME in $AUTOSTART ; do + if test -e $CONFIG_DIR/$NAME.conf ; then + ln -s "$SERVICEFILE" "$WANTDIR/logtracker-agent@$NAME.service" + fi + done +fi + +exit 0 + diff --git a/debian/logtracker-agent.README.Debian b/debian/logtracker-agent.README.Debian new file mode 100644 index 0000000..a1b5c99 --- /dev/null +++ b/debian/logtracker-agent.README.Debian @@ -0,0 +1,26 @@ +After logtracker-server has been installed, run the following commands on the server to configure agents access: + + sudo -u postgres createuser logtracker_agent; + echo "ALTER USER logtracker_agent WITH PASSWORD 'new_password';" | sudo -u postgres psql + echo 'GRANT CONNECT ON DATABASE logtracker TO logtracker_agent;' | sudo -u logtracker psql + echo 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO logtracker_agent;' | sudo -u logtracker psql + echo 'GRANT INSERT ON collection_entry TO logtracker_agent;' | sudo -u logtracker psql + +To configure a logtracker agent add a /etc/logtracker/agents/SOME-FILENAME.conf with: + +[DEFAULT] +host = dummy-server.org +database = logtracker +user = logtracker +password = new_password +agent = journald +machine = # when targeting a container + +or: + +[DEFAULT] +host = dummy-server.org +database = logtracker +user = logtracker +password = new_password +agent = exim diff --git a/debian/logtracker-agent.dirs b/debian/logtracker-agent.dirs new file mode 100644 index 0000000..03cf0fb --- /dev/null +++ b/debian/logtracker-agent.dirs @@ -0,0 +1 @@ +/etc/logtracker/ diff --git a/debian/logtracker-agent.install b/debian/logtracker-agent.install new file mode 100644 index 0000000..5277465 --- /dev/null +++ b/debian/logtracker-agent.install @@ -0,0 +1 @@ +debian/logtracker-agent-generator /lib/systemd/system-generators diff --git a/debian/logtracker-agent.service b/debian/logtracker-agent.service new file mode 100644 index 0000000..ef202df --- /dev/null +++ b/debian/logtracker-agent.service @@ -0,0 +1,15 @@ +# This service is actually a systemd target, +# but we are using a service since targets cannot be reloaded. + +[Unit] +Description=Logtracker agent service +After=network.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/true +ExecReload=/bin/true + +[Install] +WantedBy=multi-user.target diff --git a/debian/logtracker-agent@.service b/debian/logtracker-agent@.service new file mode 100644 index 0000000..2afd558 --- /dev/null +++ b/debian/logtracker-agent@.service @@ -0,0 +1,19 @@ +[Unit] +Description=Logtracker agent %i +PartOf=logtracker-agent.service +ReloadPropagatedFrom=logtracker-agent.service +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +ExecStart=/usr/bin/python3 /usr/lib/python3/dist-packages/logtracker/agent/agent.py /etc/logtracker/agents/%i.conf +PIDFile=/run/logtracker/agent-%i.pid +RuntimeDirectory=logtracker +KillMode=process +ExecReload=/bin/kill -HUP $MAINPID +RestartSec=5s +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/debian/logtracker-manage b/debian/logtracker-manage new file mode 100755 index 0000000..ce43d2e --- /dev/null +++ b/debian/logtracker-manage @@ -0,0 +1,22 @@ +#!/bin/sh + +NAME=logtracker +MANAGE=/usr/lib/$NAME/manage.py + +# check user +if test x$1 = x"--forceuser" +then + shift +elif test $(id -un) != "$NAME" +then + echo "error: must use $0 with user ${NAME}" + exit 1 +fi + +if test $# -eq 0 +then + python3 ${MANAGE} help + exit 1 +fi + +python3 ${MANAGE} "$@" diff --git a/debian/logtracker.dirs b/debian/logtracker.dirs new file mode 100644 index 0000000..3f0d4a4 --- /dev/null +++ b/debian/logtracker.dirs @@ -0,0 +1,4 @@ +/etc/logtracker +/usr/lib/logtracker +/var/lib/logtracker/collectstatic +/var/log/logtracker diff --git a/debian/logtracker.install b/debian/logtracker.install new file mode 100644 index 0000000..0633b61 --- /dev/null +++ b/debian/logtracker.install @@ -0,0 +1,4 @@ +debian/logtracker-manage /usr/bin +debian/settings.py /etc/logtracker +debian/uwsgi.ini /etc/logtracker +debian/nginx/logtracker-example.conf /etc/nginx/sites-available diff --git a/debian/logtracker.postinst b/debian/logtracker.postinst new file mode 100644 index 0000000..9525d41 --- /dev/null +++ b/debian/logtracker.postinst @@ -0,0 +1,48 @@ +#! /bin/sh + +set -e + +NAME="logtracker" +USER=$NAME +GROUP=$NAME +CONFIG_DIR="/etc/$NAME" +MANAGE_SCRIPT="/usr/bin/$NAME-manage" + +case "$1" in + configure) + + # make sure the administrative user exists + if ! getent passwd $USER >/dev/null; then + adduser --disabled-password --quiet --system \ + --no-create-home --home /var/lib/$NAME \ + --gecos "$NAME user" --group $USER + fi + getent group logtracker | grep adm || usermod -a -G adm logtracker + # ensure dirs ownership + chown $USER:$GROUP /var/log/$NAME + chown $USER:$GROUP /var/lib/$NAME/collectstatic + # create a secret file + SECRET_FILE=$CONFIG_DIR/secret + if [ ! -f $SECRET_FILE ]; then + echo -n "Generating Django secret..." >&2 + cat /dev/urandom | tr -dc [:alnum:]-_\!\%\^:\; | head -c70 > $SECRET_FILE + chown root:$GROUP $SECRET_FILE + chmod 0440 $SECRET_FILE + fi + + # Create role and database + sudo -u postgres psql -lqt | cut -d \| -f 1 | grep -q logtracker || (sudo -u postgres createuser logtracker; sudo -u postgres createdb -O logtracker logtracker;) + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/logtracker.service b/debian/logtracker.service new file mode 100644 index 0000000..06b505a --- /dev/null +++ b/debian/logtracker.service @@ -0,0 +1,21 @@ +[Unit] +Description=logtracker +After=network.target syslog.target postgresql.service +Wants=postgresql.service + +[Service] +Type=notify +Environment=LANG=C.UTF-8 +User=%p +Group=%p +ExecStartPre=/usr/bin/logtracker-manage migrate --noinput +ExecStartPre=/usr/bin/logtracker-manage collectstatic --noinput +ExecStart=/usr/bin/uwsgi --ini /etc/%p/uwsgi.ini +ExecReload=/bin/kill -HUP $MAINPID +KillSignal=SIGQUIT +PrivateTmp=true +Restart=on-failure +RuntimeDirectory=logtracker + +[Install] +WantedBy=multi-user.target diff --git a/debian/nginx/logtracker-example.conf b/debian/nginx/logtracker-example.conf new file mode 100644 index 0000000..e548e68 --- /dev/null +++ b/debian/nginx/logtracker-example.conf @@ -0,0 +1,48 @@ +server { + listen 443; + server_name _; + + ssl on; + ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem; + ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key; + + access_log /var/log/nginx/logtracker.example.org-access.log combined; + error_log /var/log/nginx/logtracker.example.org-error.log; + + location ~ ^/static/(.+)$ { + root /; + try_files /var/lib/logtracker/collectstatic/$1 + =404; + } + + location / { + proxy_pass http://unix:/var/run/logtracker/logtracker.sock; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-SSL on; + proxy_set_header X-Forwarded-Protocol ssl; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} + +server { + listen 80; + server_name _; + + access_log /var/log/nginx/logtracker.example.org-access.log combined; + error_log /var/log/nginx/logtracker.example.org-error.log; + + location ~ ^/static/(.+)$ { + root /; + try_files /var/lib/logtracker/collectstatic/$1 + =404; + } + + location / { + proxy_pass http://unix:/var/run/logtracker/logtracker.sock; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} diff --git a/debian/python3-logtracker.dirs b/debian/python3-logtracker.dirs new file mode 100644 index 0000000..d5422ca --- /dev/null +++ b/debian/python3-logtracker.dirs @@ -0,0 +1 @@ +/usr/lib/logtracker diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..693f1ee --- /dev/null +++ b/debian/rules @@ -0,0 +1,14 @@ +#!/usr/bin/make -f +# See debhelper(7) (uncomment to enable) +# output every command that modifies files on the build system. +#export DH_VERBOSE = 1 + +export PYBUILD_NAME=logtracker +export PYBUILD_INSTALL_ARGS_python3=--install-scripts=/usr/lib/logtracker/ + +%: + dh $@ --with python3 --buildsystem=pybuild + + +override_dh_auto_test: + DJANGO_SETTINGS_MODULE=tests.settings pytest-3 --disable-pytest-warnings diff --git a/debian/settings.py b/debian/settings.py new file mode 100644 index 0000000..9298e64 --- /dev/null +++ b/debian/settings.py @@ -0,0 +1,37 @@ +ALLOWED_HOSTS = ['logtracker.entrouvert.org'] + +INSTALLED_APPS += [ + 'mellon' +] + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'USER': 'logtracker', + 'NAME': 'logtracker', + } +} + + +AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', + 'mellon.backends.SAMLBackend') + +LOGIN_URL = 'mellon_login' +LOGOUT_URL = 'mellon_logout' + +MELLON_ATTRIBUTE_MAPPING = { + 'username': '{attributes[username][0]}', + 'email': '{attributes[email][0]}', + 'first_name': '{attributes[first_name][0]}', + 'last_name': '{attributes[last_name][0]}', + } + + +MELLON_SUPERUSER_MAPPING = {'is_superuser': (u'true',)} +MELLON_USERNAME_TEMPLATE = '{attributes[username][0]}' + +MELLON_PUBLIC_KEYS = ['/etc/logtracker/saml.crt'] +MELLON_PRIVATE_KEY = '/etc/logtracker/saml.key' +MELLON_IDENTITY_PROVIDERS = [ + {'METADATA': '/etc/logtracker/idp-metadata.xml',}, + ] diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/uwsgi.ini b/debian/uwsgi.ini new file mode 100644 index 0000000..077d5cc --- /dev/null +++ b/debian/uwsgi.ini @@ -0,0 +1,7 @@ +[uwsgi] +plugin = python3 +http-socket = /run/logtracker/logtracker.sock +#--wsgi-file logtracker.wsgi.py +module = logtracker.wsgi:application +chmod-socket = 666 +vacuum = true diff --git a/logtracker/__init__.py b/logtracker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logtracker/agent/__init__.py b/logtracker/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logtracker/agent/agent.py b/logtracker/agent/agent.py new file mode 100644 index 0000000..473ffe8 --- /dev/null +++ b/logtracker/agent/agent.py @@ -0,0 +1,44 @@ +#!/usr/bin/python3 +# Entrouvert 2019 +import configparser +import importlib +import io +import json +import shlex +import sys +import psycopg2 +import subprocess + + +def get_config(): + configfile = sys.argv[1] + config = configparser.ConfigParser() + config.read(configfile) + return config['DEFAULT'] + + +def tail(filename): + p = subprocess.Popen(shlex.split('tail -f %s' % filename), stdout=subprocess.PIPE) + for line in io.TextIOWrapper(p.stdout): + yield line + + +class Database: + def __init__(self, host, database, user, password, **kwargs): + self.conn = psycopg2.connect(host=host, database=database, user=user, password=password) + + def write(self, entry): + with self.conn.cursor() as cursor: + json_data = json.dumps(entry['data']) + sql = 'insert into collection_entry (host, service, priority, timestamp, data) values (%s, %s, %s, %s, %s)' + args = (entry['host'], entry['service'], entry['priority'], entry['timestamp'], json_data) + cursor.execute(sql, args) + self.conn.commit() + + +if __name__ == '__main__': + config = get_config() + database = Database(**config) + agent = importlib.import_module(config['agent']) + for entry in getattr(agent, 'main')(): + database.write(entry) diff --git a/logtracker/agent/exim.py b/logtracker/agent/exim.py new file mode 100755 index 0000000..d093e9a --- /dev/null +++ b/logtracker/agent/exim.py @@ -0,0 +1,42 @@ +#!/usr/bin/python3 +# Entrouvert 2019 +# Exim log parser +# See Summary of Fields in Log Lines in https://www.exim.org/exim-html-current/doc/html/spec_html/ch-log_files.html +import socket +import re +import datetime +import pytz +from django.utils import timezone + +from logtracker.agent.agent import tail + +host = socket.getfqdn() +paris = pytz.timezone('Europe/Paris') +patterns = {'ignore': re.compile('([\d-]+) ([\d:]+) .*(Start queue run|End queue run|daemon started|relay not permitted|Spool file is locked|Connection refused|Connection timed out|no immediate delivery|error ignored|Greylisting in action|Remote host closed connection|No route to host|SMTP error|SMTP protocol error|SMTP protocol synchronization error|SMTP command timeout|no host name found|unexpected disconnection|TLS error|log string overflowed|cancelled by timeout).*'), + 'match': re.compile('(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) ([\w\d\-]+) (<=|=>|->|==|\*\*|Completed|SMTP error|Message is frozen|Frozen|Unfrozen)\s*(.*)$'), + } + + +def parse_date(string): + stamp = datetime.datetime.strptime(string, '%Y-%m-%d %H:%M:%S') + return timezone.make_aware(stamp, paris) + + +def parse_line(line): + match = re.match(patterns['match'], line) + if match: + stamp, identifier, action, raw = match.groups() + stamp = parse_date(stamp) + data = {'raw': '%s %s' % (action, raw[:511].replace("'", '')), 'identifier': identifier} + return {'host': host, 'service': 'exim', 'timestamp': stamp, 'priority': 6, 'data': data} + else: + match = re.match(patterns['ignore'], line) + if not match: + print('Failed to parse line: %s' % line) + + +def main(): + for line in tail('/var/log/exim4/mainlog'): + match = parse_line(line) + if match: + yield match diff --git a/logtracker/agent/journald.py b/logtracker/agent/journald.py new file mode 100755 index 0000000..e3fc992 --- /dev/null +++ b/logtracker/agent/journald.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +import io +import json +import shlex +import socket +import subprocess +import time + +host = socket.getfqdn() + + +def read_from_journald(): + p = subprocess.Popen(shlex.split('journalctl -o json -f'), stdout=subprocess.PIPE) + for line in io.TextIOWrapper(p.stdout): + yield line + + +def parse_line(line): + data = json.loads(line) + timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(data['__REALTIME_TIMESTAMP']) / 1000000)) + return {'host': host, + 'service': 'journald', + 'timestamp': timestamp, + 'priority': data['PRIORITY'], + 'data': data} + + +def main(): + for line in read_from_journald(): + yield parse_line(line) diff --git a/logtracker/collection/__init__.py b/logtracker/collection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logtracker/collection/admin.py b/logtracker/collection/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/logtracker/collection/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/logtracker/collection/migrations/0001_initial.py b/logtracker/collection/migrations/0001_initial.py new file mode 100644 index 0000000..e5f7fb9 --- /dev/null +++ b/logtracker/collection/migrations/0001_initial.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2019-11-12 13:58 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Entry', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('host', models.CharField(max_length=128)), + ('service', models.CharField(max_length=32)), + ('priority', models.IntegerField(choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7')], default=6)), + ('timestamp', models.DateTimeField()), + ('data', django.contrib.postgres.fields.jsonb.JSONField(blank=True)), + ], + options={ + 'ordering': ['-timestamp', '-id'], + }, + ), + ] diff --git a/logtracker/collection/migrations/__init__.py b/logtracker/collection/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logtracker/collection/models.py b/logtracker/collection/models.py new file mode 100644 index 0000000..2ae77c1 --- /dev/null +++ b/logtracker/collection/models.py @@ -0,0 +1,16 @@ +from django.db import models +from django.contrib.postgres.fields import JSONField + + +class Entry(models.Model): + host = models.CharField(max_length=128) + service = models.CharField(max_length=32) + priority = models.IntegerField(choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7')], default=6) + timestamp = models.DateTimeField() + data = JSONField(blank=True) + + class Meta: + ordering = ['-timestamp', '-id'] + + def __str__(self): + return '%s %s %s %s' % (self.timestamp, self.host, self.service, self.data) diff --git a/logtracker/collection/templates/base.html b/logtracker/collection/templates/base.html new file mode 100644 index 0000000..e9720cc --- /dev/null +++ b/logtracker/collection/templates/base.html @@ -0,0 +1,12 @@ + + + + {% block title %}Mailtracker{% endblock %} + + + +
+ {% block content %}{% endblock %} +
+ + diff --git a/logtracker/collection/templates/entries/home.html b/logtracker/collection/templates/entries/home.html new file mode 100644 index 0000000..286e612 --- /dev/null +++ b/logtracker/collection/templates/entries/home.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block content %} +

Mailtracker

+ +

Search by recipient

+ +
{% csrf_token %} + Address: + + Sender + Recipient
+ +
+ +Recent senders list + +{% endblock %} diff --git a/logtracker/collection/templates/entries/sender_list.html b/logtracker/collection/templates/entries/sender_list.html new file mode 100644 index 0000000..b207cc9 --- /dev/null +++ b/logtracker/collection/templates/entries/sender_list.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} + +{% block content %} +

Mailtracker

+ +

Recent senders

+ + + + + + + + + + +{% for sender in object_list %} + + + + + + +{% empty %} + No data yet. +{% endfor %} + +
SenderEmails CountWith errorsPending
{{ sender }}{{ sender.email_count }}{{ sender.error_count }}{{ sender.pending_count }}
+{% endblock %} diff --git a/logtracker/collection/views.py b/logtracker/collection/views.py new file mode 100644 index 0000000..b45b21b --- /dev/null +++ b/logtracker/collection/views.py @@ -0,0 +1,28 @@ +from django.contrib.auth.mixins import LoginRequiredMixin +from django.core import serializers +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import get_object_or_404 +from django.urls import reverse +from django.views.generic import TemplateView +from django.views.generic.list import ListView + +from logtracker.collection.models import Entry + + +class EntriesList(LoginRequiredMixin, ListView): + def get_queryset(self): + qs = Entry.objects.all() + return qs[:100] + + def get(self, request, *args, **kwargs): + queryset = self.get_queryset() + response = serializers.serialize("json", queryset) + return HttpResponse(response, content_type='application/json') + + +class HomeView(LoginRequiredMixin, TemplateView): + template_name = 'entries/home.html' + + def post(self, request, *args, **kwargs): + url = '%s?%s' % (reverse('emails'), '%s=%s' % (request.POST['field'], request.POST['address'])) + return HttpResponseRedirect(url) diff --git a/logtracker/mail/__init__.py b/logtracker/mail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logtracker/mail/admin.py b/logtracker/mail/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/logtracker/mail/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/logtracker/mail/apps.py b/logtracker/mail/apps.py new file mode 100644 index 0000000..6522bc5 --- /dev/null +++ b/logtracker/mail/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class CollectorConfig(AppConfig): + name = 'collector' diff --git a/logtracker/mail/management/__init__.py b/logtracker/mail/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logtracker/mail/management/commands/__init__.py b/logtracker/mail/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logtracker/mail/management/commands/cleanup.py b/logtracker/mail/management/commands/cleanup.py new file mode 100644 index 0000000..b8310cd --- /dev/null +++ b/logtracker/mail/management/commands/cleanup.py @@ -0,0 +1,18 @@ +import datetime +from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone + +from logtracker.data.models import Mail + +class Command(BaseCommand): + + def add_arguments(self, parser): + parser.add_argument('-d', '--days') + + def handle(self, *args, **options): + if options.get('days'): + days = options.get('days') + else: + days = 3 + delay = timezone.now() - datetime.timedelta(days=days) + Mail.objects.filter(stamp__lt=delay).delete() diff --git a/logtracker/mail/management/commands/collect.py b/logtracker/mail/management/commands/collect.py new file mode 100644 index 0000000..95bdcc8 --- /dev/null +++ b/logtracker/mail/management/commands/collect.py @@ -0,0 +1,25 @@ +from django.core.management.base import BaseCommand, CommandError +from logtracker.data.models import Entry, Mail, Sender +from logtracker.collectors import parse +from logtracker.utils import lock + + +class Command(BaseCommand): + help = 'Collect entries from exim4 log' + + def add_arguments(self, parser): + parser.add_argument('-l', '--logfile', default='/var/log/exim4/mainlog') + parser.add_argument('--lockfile', default='/var/lock/mainlog.lock') + + def handle(self, logfile, *args, **options): + with lock(options.get('lockfile')): + new_mails = [] + for stamp, identifier, action, data in parse(logfile): + mail, created = Mail.objects.get_or_create(identifier=identifier) + entry = Entry(mail=mail, action=action, data=data, stamp=stamp) + entry.save() + if created: + new_mails.append(mail) + + for mail in new_mails: + mail.handle_exim_data() diff --git a/logtracker/mail/management/commands/errors.py b/logtracker/mail/management/commands/errors.py new file mode 100644 index 0000000..79b6ed8 --- /dev/null +++ b/logtracker/mail/management/commands/errors.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand, CommandError +from logtracker.data.models import Mail + +class Command(BaseCommand): + help = 'Collect entries from exim4 log' + + def handle(self, *args, **options): + for mail in Mail.objects.filter(has_error=True): + print(mail) diff --git a/logtracker/mail/management/commands/get_entries.py b/logtracker/mail/management/commands/get_entries.py new file mode 100644 index 0000000..1d4f8d9 --- /dev/null +++ b/logtracker/mail/management/commands/get_entries.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand, CommandError +from logtracker.data.models import Mail + +class Command(BaseCommand): + help = 'Collect entries from exim4 log' + + def add_arguments(self, parser): + parser.add_argument('sender', type=str) + + def handle(self, *args, **options): + for entry in Mail.objects.filter(sender=options['sender']): + print(entry) diff --git a/logtracker/mail/management/commands/senders_count.py b/logtracker/mail/management/commands/senders_count.py new file mode 100644 index 0000000..0dea51c --- /dev/null +++ b/logtracker/mail/management/commands/senders_count.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand, CommandError +from logtracker.data.models import Sender + +class Command(BaseCommand): + help = 'Collect entries from exim4 log' + + def handle(self, *args, **options): + for sender in Sender.objects.all(): + print("%s %s" % (sender, sender.emails_count)) diff --git a/logtracker/mail/models.py b/logtracker/mail/models.py new file mode 100644 index 0000000..37498cf --- /dev/null +++ b/logtracker/mail/models.py @@ -0,0 +1,84 @@ +from django.db import models +import json + +# Create your models here. + +TYPES = [('<=', 'Arrival'), + ('=>', 'Delivery'), + ('==', 'Defered'), + ('**', 'Failed'), + ('->', 'Additional'), + ('*>', 'Suppressed'), + ('Completed', 'Completed'), + ('Frozen', 'Frozen'), + ('SMTP error', 'SMTP error'), + ] + + +class Address(models.Model): + email = models.EmailField(blank=True, null=True) + + class Meta: + abstract = True + ordering = ['email'] + + def email_count(self): + return self.mail_set.all().count() + + def pending_count(self): + return self.mail_set.filter(has_completed=False).count() + + def error_count(self): + return self.mail_set.filter(has_error=True).count() + + def __str__(self): + return self.email + + +class Sender(Address): + pass + + +class Recipient(Address): + pass + + +class Mail(models.Model): + identifier = models.CharField(max_length=30) + sender = models.ForeignKey(Sender, null=True) + recipients = models.ManyToManyField(Recipient) + has_error = models.BooleanField(default=False) + has_completed = models.BooleanField(default=False) + stamp = models.DateTimeField(null=True) + + def __str__(self): + return '%s %s' % (self.identifier, self.sender) + + def handle_exim_data(self): + for entry in self.entry_set.all(): + if entry.action == '<=': + address = entry.data.split(' ')[0] + sender, _ = Sender.objects.get_or_create(email=address) + self.sender = sender + self.stamp = entry.stamp + elif entry.action in ('=>', '->'): + address, _ = Recipient.objects.get_or_create(email=entry.data.split(' ')[0]) + self.recipients.add(address) + elif entry.action == 'Completed': + self.has_completed = True + elif entry.action == '==' or entry.action == '**': + self.has_error = True + self.save() + + +class Entry(models.Model): + mail = models.ForeignKey(Mail) + data = models.CharField(max_length=512) + action = models.CharField(max_length=30, choices=TYPES) + stamp = models.DateTimeField() + + class Meta: + ordering = ['stamp', 'id'] + + def __str__(self): + return '%s %s %s' % (self.stamp, self.action, self.data) diff --git a/logtracker/mail/templates/base.html b/logtracker/mail/templates/base.html new file mode 100644 index 0000000..e9720cc --- /dev/null +++ b/logtracker/mail/templates/base.html @@ -0,0 +1,12 @@ + + + + {% block title %}Mailtracker{% endblock %} + + + +
+ {% block content %}{% endblock %} +
+ + diff --git a/logtracker/mail/templates/entries/home.html b/logtracker/mail/templates/entries/home.html new file mode 100644 index 0000000..286e612 --- /dev/null +++ b/logtracker/mail/templates/entries/home.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block content %} +

Mailtracker

+ +

Search by recipient

+ +
{% csrf_token %} + Address: + + Sender + Recipient
+ +
+ +Recent senders list + +{% endblock %} diff --git a/logtracker/mail/templates/entries/sender_list.html b/logtracker/mail/templates/entries/sender_list.html new file mode 100644 index 0000000..b207cc9 --- /dev/null +++ b/logtracker/mail/templates/entries/sender_list.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} + +{% block content %} +

Mailtracker

+ +

Recent senders

+ + + + + + + + + + +{% for sender in object_list %} + + + + + + +{% empty %} + No data yet. +{% endfor %} + +
SenderEmails CountWith errorsPending
{{ sender }}{{ sender.email_count }}{{ sender.error_count }}{{ sender.pending_count }}
+{% endblock %} diff --git a/logtracker/mail/views.py b/logtracker/mail/views.py new file mode 100644 index 0000000..72bb2e8 --- /dev/null +++ b/logtracker/mail/views.py @@ -0,0 +1,48 @@ +from urllib.parse import unquote +from django.views.generic import TemplateView +from django.views.generic.list import ListView +from django.http import JsonResponse, HttpResponseRedirect +from django.contrib.auth.mixins import LoginRequiredMixin +from django.urls import reverse +from django.shortcuts import get_object_or_404 +from logtracker.mail.models import Mail, Sender, Recipient + + +def check(val): + return val == 'true' + + +class EmailsList(LoginRequiredMixin, ListView): + def get_queryset(self): + if self.request.GET.get('from'): + sender = get_object_or_404(Sender, email=unquote(self.request.GET.get('from'))) + qs = sender.mail_set.all() + elif self.request.GET.get('to'): + recipient = get_object_or_404(Recipient, email=unquote(self.request.GET.get('to'))) + qs = recipient.mail_set.all() + else: + qs = Mail.objects.all() + if self.request.GET.get('has_error'): + qs = qs.filter(has_error=check(self.request.GET.get('has_error'))) + if self.request.GET.get('has_completed'): + qs = qs.filter(has_completed=check(self.request.GET.get('has_completed'))) + return qs[:100] + + def get(self, request, *args, **kwargs): + queryset = self.get_queryset() + data = {m.identifier: [str(e) for e in m.entry_set.all()] for m in queryset} + return JsonResponse({'data': data}) + + +class MailHome(LoginRequiredMixin, TemplateView): + template_name = 'entries/home.html' + form_class = Recipient + + def post(self, request, *args, **kwargs): + url = '%s?%s' % (reverse('emails'), '%s=%s' % (request.POST['field'], request.POST['address'])) + return HttpResponseRedirect(url) + + +class SendersList(LoginRequiredMixin, ListView): + + model = Sender diff --git a/logtracker/settings.py b/logtracker/settings.py new file mode 100644 index 0000000..31bc7a4 --- /dev/null +++ b/logtracker/settings.py @@ -0,0 +1,127 @@ +""" +Django settings for logtracker project. + +Generated by 'django-admin startproject' using Django 1.11.18. + +For more information on this file, see +https://docs.djangoproject.com/en/1.11/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.11/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'dxk)c9u(gq7@2tb)+*!=$09c=o0qnwbt^vrrmjcx-bsl&2!u7b' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'logtracker.collection', + 'logtracker.mail', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'logtracker.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'logtracker.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.11/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.11/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.11/howto/static-files/ + +STATIC_URL = '/static/' +STATIC_ROOT = '/var/lib/logtracker/collectstatic' + +local_settings_file = '/etc/logtracker/settings.py' +if os.path.exists(local_settings_file): + exec(open(local_settings_file).read()) diff --git a/logtracker/urls.py b/logtracker/urls.py new file mode 100644 index 0000000..d874ccd --- /dev/null +++ b/logtracker/urls.py @@ -0,0 +1,18 @@ +from django.conf.urls import include, url +from django.contrib import admin +from django.conf import settings + +from logtracker.collection.views import EntriesList, HomeView +from logtracker.mail.views import EmailsList, SendersList, MailHome + +urlpatterns = [ + url(r'^admin/', admin.site.urls), + url(r'^$', HomeView.as_view()), + url(r'^api/collection/$', EntriesList.as_view()), + url(r'^api/mail/$', EmailsList.as_view(), name='emails'), + url(r'^mail/$', MailHome.as_view(), name='mail'), + url(r'^mail/senders/$', SendersList.as_view(), name='senders'), +] + +if 'mellon' in settings.INSTALLED_APPS: + urlpatterns += url(r'^accounts/mellon/', include('mellon.urls')), diff --git a/logtracker/utils.py b/logtracker/utils.py new file mode 100644 index 0000000..e651aec --- /dev/null +++ b/logtracker/utils.py @@ -0,0 +1,15 @@ +import os +import sys +import fcntl +from contextlib import contextmanager +from io import BlockingIOError + +@contextmanager +def lock(lockfile): + try: + handle = open(lockfile, 'w') + fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + except BlockingIOError: + print('locked') + sys.exit(1) diff --git a/logtracker/wsgi.py b/logtracker/wsgi.py new file mode 100644 index 0000000..0de2857 --- /dev/null +++ b/logtracker/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for logtracker project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "logtracker.settings") + +application = get_wsgi_application() diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..d3b6e98 --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "logtracker.settings") + try: + from django.core.management import execute_from_command_line + except ImportError: + # The above import may fail for some other reason. Ensure that the + # issue is really that Django is missing to avoid masking other + # exceptions on Python 2. + try: + import django + except ImportError: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) + raise + execute_from_command_line(sys.argv) diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..ba83155 --- /dev/null +++ b/setup.py @@ -0,0 +1,134 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import subprocess +import shutil +import sys + +from setuptools.command.install_lib import install_lib as _install_lib +from distutils.command.build import build as _build +from setuptools.command.sdist import sdist +from distutils.cmd import Command +from setuptools import setup, find_packages + + +class deb(sdist): + def run(self, *args): + os.system('git clean -fdx') + version = get_version() + sdist.run(self) + shutil.move("dist/logtracker-%s.tar.gz" % version, '../logtracker_%s.orig.tar.gz' % version) + os.system('git clean -fdx') + os.system('dpkg-source -b .') + os.system('sudo HOME=$HOME DIST=buster cowbuilder --build ../logtracker_%s-1.dsc' % version) + + +class eo_sdist(sdist): + def run(self): + if os.path.exists('VERSION'): + os.remove('VERSION') + version = get_version() + version_file = open('VERSION', 'w') + version_file.write(version) + version_file.close() + sdist.run(self) + if os.path.exists('VERSION'): + os.remove('VERSION') + + +def get_version(): + '''Use the VERSION, if absent generates a version with git describe, if not + tag exists, take 0.0- and add the length of the commit log. + ''' + if os.path.exists('VERSION'): + with open('VERSION', 'r') as v: + return v.read() + if os.path.exists('.git'): + p = subprocess.Popen(['git','describe','--dirty=.dirty','--match=v*'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result = p.communicate()[0] + if p.returncode == 0: + result = result.decode('ascii').strip()[1:] # strip spaces/newlines and initial v + if '-' in result: # not a tagged version + real_number, commit_count, commit_hash = result.split('-', 2) + version = '%s.post%s+%s' % (real_number, commit_count, commit_hash) + else: + version = result + return version + else: + return '0.0.post%s' % len( + subprocess.check_output( + ['git', 'rev-list', 'HEAD']).splitlines()) + return '0.0' + + +class compile_translations(Command): + description = 'compile message catalogs to MO files via django compilemessages' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + from django.core.management import call_command + for path, dirs, files in os.walk('fargo'): + if 'locale' not in dirs: + continue + curdir = os.getcwd() + os.chdir(os.path.realpath(path)) + call_command('compilemessages') + os.chdir(curdir) + except ImportError: + sys.stderr.write('!!! Please install Django >= 1.4 to build translations\n') + + +class build(_build): + sub_commands = [('compile_translations', None)] + _build.sub_commands + + +class install_lib(_install_lib): + def run(self): + self.run_command('compile_translations') + _install_lib.run(self) + + +setup( + name='logtracker', + version=get_version(), + description='Exim Logger', + long_description=open('README').read(), + author="Entr'ouvert", + author_email='info@entrouvert.com', + packages=find_packages(), + include_package_data=True, + scripts=('manage.py',), + url='https://dev.entrouvert.org/projects/logtracker/', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Environment :: Web Environment', + 'Framework :: Django', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + ], + install_requires=[ + 'django>=1.11', + 'psycopg2', + 'django-mellon' + ], + zip_safe=False, + cmdclass={ + 'build': build, + 'compile_translations': compile_translations, + 'install_lib': install_lib, + 'sdist': eo_sdist, + 'deb': deb, + }, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7a14a95 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,48 @@ +import json +import os +import pytest +import psycopg2 +from django.db import connection +from django.contrib.auth.models import User + +from logtracker.agent import journald, exim + + +basepath = os.path.dirname(__file__) + + +def run_sql(sql): + conn = psycopg2.connect(database='postgres') + cur = conn.cursor() + cur.execute(sql) + conn.close() + + +@pytest.fixture +def journald_data(request, db, client): + with open('%s/testdata.journald' % basepath) as fh: + for line in fh.readlines(): + parsed = journald.parse_line(line) + write(parsed) + + +def write(entry): + with connection.cursor() as c: + entry['data'] = json.dumps(entry['data']) + c.execute('''insert into collection_entry (host, service, priority, timestamp, data) values (%s, %s, %s, %s, %s)''', + (entry['host'], entry['service'], entry['priority'], entry['timestamp'], entry['data'])) + + +@pytest.fixture +def exim_data(request, db, client): + with open('%s/testdata.exim' % basepath) as fh: + for line in fh.readlines(): + parsed = exim.parse_line(line) + if parsed: + write(parsed) + + +@pytest.fixture +def auth(db, client): + User.objects.create_user(username='john', password='Doe') + client.login(username='john', password='Doe') diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..8f5e607 --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,123 @@ +""" +Django settings for logtracker project. + +Generated by 'django-admin startproject' using Django 1.11.18. + +For more information on this file, see +https://docs.djangoproject.com/en/1.11/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.11/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'dxk)c9u(gq7@2tb)+*!=$09c=o0qnwbt^vrrmjcx-bsl&2!u7b' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'logtracker.collection', + 'logtracker.mail' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'logtracker.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'logtracker.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.11/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.11/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.11/howto/static-files/ + +STATIC_URL = '/static/' +STATIC_ROOT = '/var/lib/logtracker/collectstatic' diff --git a/tests/test_agent_journald.py b/tests/test_agent_journald.py new file mode 100644 index 0000000..227fcce --- /dev/null +++ b/tests/test_agent_journald.py @@ -0,0 +1,5 @@ +from logtracker.collection.models import Entry + + +def test_import_data(journald_data): + assert len(Entry.objects.all()) == 10 diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 0000000..3d92e42 --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,20 @@ +import pytest +import json + + +def test_auth(client, monkeypatch): + resp = client.get('/api/mail/') + assert resp.status_code == 302 + + +def test_home_page(client, auth): + page = client.get('/') + assert page.status_code == 200 + + +def test_entries_list(auth, client, journald_data): + page = client.get('/api/collection/') + assert page.status_code == 200 + + entries = json.loads(page.content.decode()) + assert len(entries) == 10 diff --git a/tests/test_mail.py b/tests/test_mail.py new file mode 100644 index 0000000..2d2bf75 --- /dev/null +++ b/tests/test_mail.py @@ -0,0 +1,17 @@ +import pytest +import json + +from django.contrib.auth.models import User + + +def test_mail_api(client, auth): + page = client.get('/api/mail/') + assert page.status_code == 200 + + +def test_entries_list(auth, client, exim_data): + page = client.get('/api/collection/') + assert page.status_code == 200 + + entries = json.loads(page.content.decode()) + assert len(entries) == 23 diff --git a/tests/testdata.exim b/tests/testdata.exim new file mode 100644 index 0000000..7c951e6 --- /dev/null +++ b/tests/testdata.exim @@ -0,0 +1,27 @@ +2019-02-09 14:12:32 1gsSQq-0004Nr-HL <= ne-pas-repondre@chicago.fr H=my.server.net [5.135.221.10] P=esmtp S=8785 id=E1gsSQq-0001GI-EW@auquo.myserver.net +2019-02-09 14:12:32 1gsSQq-0004Nr-HL == john.guyot33@sfr.fr R=dnslookup T=remote_smtp defer (-53): retry time not reached for any host for 'sfr.fr' +2019-02-09 14:13:11 1gsSRS-0004Nu-Mk => ymelin@gmail.com R=dnslookup T=remote_smtp H=alt1.gmail-smtp-in.l.google.com [64.233.164.26] X=TLS1.2:ECDHE_RSA_CHACHA20_POLY1305:256 CV=yes DN="C=US,ST=California,L=Mountain View,O=Google LLC,CN=mx.google.com" C="250 2.0.0 OK 1549717991 u2-v6si1114219ljg.59 - gsmtp" +2019-02-09 14:13:11 1gsSRS-0004Nu-Mk Completed +2019-02-09 14:14:12 1gsSSS-0004Nz-6i <= ne-pas-repondre@departement666.fr H=my.server.net [5.135.221.10] P=esmtp S=7977 id=20190209131412.26403.28082@authentic.node1.prod.saas.myserver.net +2019-02-09 14:14:12 1gsSSS-0004Nz-6i ** dodu.jc06@hotmail.fr R=dnslookup T=remote_smtp H=eur.olc.protection.outlook.com [104.47.5.33] X=TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=yes DN="C=US,ST=Washington,L=Redmond,O=Microsoft Corporation,CN=mail.protection.outlook.com": SMTP error from remote mail server after RCPT TO:: 550 5.5.0 Requested action not taken: mailbox unavailable. +2019-02-09 14:14:12 1gsSSS-0004O2-PV <= <> R=1gsSSS-0004Nz-6i U=Debian-exim P=local S=9550 +2019-02-09 14:14:12 1gsSSS-0004Nz-6i Completed +2019-02-09 14:14:13 1gsSSS-0004O2-PV ** ne-pas-repondre@departement666.fr R=dnslookup T=remote_smtp H=mail-av5.departement666.fr [193.48.79.5]: SMTP error from remote mail server after RCPT TO:: 550 5.1.1 : Recipient address rejected: User unknown in relay recipient table +2019-02-09 14:14:13 1gsSSS-0004O2-PV Frozen (delivery error message) +2019-02-09 14:15:45 1gsSTx-0004OW-B7 <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=8433 id=20190209131545.4912.74435@authentic.myserver.net +2019-02-09 14:15:46 1gsSTx-0004OW-B7 => emmanuel@asfar.net R=dnslookup T=remote_smtp H=a.mx.asfar.net [69.168.84.3] C="250 ok 1549718146 qp 68658" +2019-02-09 14:15:46 1gsSTx-0004OW-B7 Completed +2019-02-09 14:18:25 1gsSWW-0004Ox-VJ <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=7971 id=20190209131824.8393.69975@authentic.myserver.net +2019-02-09 14:18:31 1gsSWd-0004P2-FY <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=8283 id=20190209131831.8393.49556@authentic.myserver.net +2019-02-09 14:18:32 1gsSWd-0004P2-FY SMTP error from remote mail server after end of data: 421-4.7.0 This message does not have authentication information or fails to pass\n421-4.7.0 authentication checks. To best protect our users from spam, the\n421-4.7.0 message has been blocked. Please visit\n421-4.7.0 https://support.google.com/mail/answer/81126#authentication for more\n421 4.7.0 information. q11si3575718wro.250 - gsmtp +2019-02-09 14:21:23 1gsSZP-0004PN-0x <= ne-pas-repondre+mymeaux@myserver.net H=my.server.net [5.135.221.10] P=esmtp S=6828 id=E1gsSZN-000x9T-ST@wcs.node1.prod.saas.myserver.net +2019-02-09 14:21:23 1gsSZP-0004PN-0x => a.desales@mars.planet R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery" +2019-02-09 14:21:23 1gsSZP-0004PN-0x -> a.obin@mars.planet R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery" +2019-02-09 14:21:23 1gsSZP-0004PN-0x -> athy.dick@mars.planet R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery" +2019-02-09 14:21:23 1gsSZP-0004PN-0x -> aurelie.rihon@mars.planet R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery" +2019-02-09 14:21:23 1gsSZP-0004PN-0x Completed +2019-02-09 17:25:30 H=(WIN-BR8CPBUF43R) [155.94.137.16] F= rejected RCPT : relay not permitted +2019-02-18 09:49:55 1gvecd-0008Vl-Jj H=netcourier.com [68.178.213.61]: Remote host closed connection in response to initial connection +2019-02-18 09:51:47 1gveaE-0008UG-HK H=alt2.gmail-smtp-in.l.google.com [2404:6800:4003:c02::1a] Connection timed out +2019-02-18 08:06:10 1gvbfc-0000qJ-BL H=mail.ualusa.com [168.235.95.126] Connection refused +2019-02-18 08:06:10 1gvcTW-0004qQ-RR Message is frozen diff --git a/tests/testdata.journald b/tests/testdata.journald new file mode 100644 index 0000000..650b02b --- /dev/null +++ b/tests/testdata.journald @@ -0,0 +1,10 @@ +{"_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","SYSLOG_PID":"5115","SYSLOG_TIMESTAMP":"Nov 8 14:45:01 ","_PID":"5115","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c406f;b=9b6948aea6564a80a6ecb7fc6684491a;m=bfcea5c9;t=596d5fd1133cf;x=1e677b790058a963","_UID":"0","_GID":"0","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","_TRANSPORT":"syslog","_SOURCE_REALTIME_TIMESTAMP":"1573220701505885","SYSLOG_IDENTIFIER":"CRON","__MONOTONIC_TIMESTAMP":"3217991113","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_HOSTNAME":"pad","__REALTIME_TIMESTAMP":"1573220701516751","SYSLOG_FACILITY":"9"} +{"__MONOTONIC_TIMESTAMP":"3217991316","_SYSTEMD_UNIT":"cron.service","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_UID":"0","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573220701516954","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4070;b=9b6948aea6564a80a6ecb7fc6684491a;m=bfcea694;t=596d5fd11349a;x=488c3f2177060430","_COMM":"cron","_AUDIT_LOGINUID":"0","_SYSTEMD_CGROUP":"/system.slice/cron.service","SYSLOG_FACILITY":"10","_SOURCE_REALTIME_TIMESTAMP":"1573220701511781","_GID":"0","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SYSTEMD_SLICE":"system.slice","SYSLOG_PID":"5114","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_TRANSPORT":"syslog","_CMDLINE":"/usr/sbin/CRON -f","PRIORITY":"6","_CAP_EFFECTIVE":"3fffffffff","MESSAGE":"pam_unix(cron:session): session closed for user root","_HOSTNAME":"pad","SYSLOG_IDENTIFIER":"CRON","_SELINUX_CONTEXT":"unconfined\n","_AUDIT_SESSION":"13","SYSLOG_TIMESTAMP":"Nov 8 14:45:01 ","_PID":"5114"} +{"MESSAGE":"Valid eCryptfs headers not found in file header region or xattr region, inode 7258970","_SOURCE_MONOTONIC_TIMESTAMP":"3779354493","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4071;b=9b6948aea6564a80a6ecb7fc6684491a;m=e1426d66;t=596d61e84fb6b;x=b17588d1ff1c6b03","PRIORITY":"7","SYSLOG_FACILITY":"0","_TRANSPORT":"kernel","SYSLOG_IDENTIFIER":"kernel","__MONOTONIC_TIMESTAMP":"3779226982","_HOSTNAME":"pad","__REALTIME_TIMESTAMP":"1573221262752619"} +{"__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4072;b=9b6948aea6564a80a6ecb7fc6684491a;m=e195117c;t=596d61ed79f82;x=a9d1e5633a803563","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","__MONOTONIC_TIMESTAMP":"3784642940","_TRANSPORT":"kernel","SYSLOG_IDENTIFIER":"kernel","PRIORITY":"7","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_HOSTNAME":"pad","SYSLOG_FACILITY":"0","MESSAGE":"Valid eCryptfs headers not found in file header region or xattr region, inode 10093410","__REALTIME_TIMESTAMP":"1573221268168578","_SOURCE_MONOTONIC_TIMESTAMP":"3784771755"} +{"PRIORITY":"6","SYSLOG_IDENTIFIER":"CRON","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","SYSLOG_FACILITY":"10","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_CAP_EFFECTIVE":"3fffffffff","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SELINUX_CONTEXT":"unconfined\n","_AUDIT_SESSION":"14","_SYSTEMD_SLICE":"system.slice","_AUDIT_LOGINUID":"0","_HOSTNAME":"pad","__MONOTONIC_TIMESTAMP":"3817997608","_UID":"0","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573221301523247","_PID":"5350","_TRANSPORT":"syslog","MESSAGE":"pam_unix(cron:session): session opened for user root by (uid=0)","SYSLOG_PID":"5350","_GID":"0","_SYSTEMD_UNIT":"cron.service","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_COMM":"cron","_SYSTEMD_CGROUP":"/system.slice/cron.service","_CMDLINE":"/usr/sbin/CRON -f","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4073;b=9b6948aea6564a80a6ecb7fc6684491a;m=e3920528;t=596d620d4932f;x=4d57d832d99b5cae","_SOURCE_REALTIME_TIMESTAMP":"1573221301523129"} +{"__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4074;b=9b6948aea6564a80a6ecb7fc6684491a;m=e3920819;t=596d620d4961f;x=f8784991133557ec","SYSLOG_PID":"5351","_SYSTEMD_UNIT":"cron.service","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","__REALTIME_TIMESTAMP":"1573221301523999","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_PID":"5351","_TRANSPORT":"syslog","_GID":"0","_SYSTEMD_CGROUP":"/system.slice/cron.service","_SELINUX_CONTEXT":"unconfined\n","_UID":"0","SYSLOG_FACILITY":"9","_HOSTNAME":"pad","_CMDLINE":"/usr/sbin/CRON -f","_CAP_EFFECTIVE":"3fffffffff","_EXE":"/usr/sbin/cron","_COMM":"cron","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_SOURCE_REALTIME_TIMESTAMP":"1573221301523811","SYSLOG_IDENTIFIER":"CRON","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_SYSTEMD_SLICE":"system.slice","_AUDIT_LOGINUID":"0","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","__MONOTONIC_TIMESTAMP":"3817998361","_AUDIT_SESSION":"14"} +{"__REALTIME_TIMESTAMP":"1573221301526069","SYSLOG_FACILITY":"10","_COMM":"cron","_SELINUX_CONTEXT":"unconfined\n","PRIORITY":"6","_SYSTEMD_SLICE":"system.slice","_SYSTEMD_CGROUP":"/system.slice/cron.service","_AUDIT_SESSION":"14","_HOSTNAME":"pad","_TRANSPORT":"syslog","_CMDLINE":"/usr/sbin/CRON -f","__MONOTONIC_TIMESTAMP":"3818000431","_SOURCE_REALTIME_TIMESTAMP":"1573221301526029","SYSLOG_PID":"5350","_CAP_EFFECTIVE":"3fffffffff","_SYSTEMD_UNIT":"cron.service","MESSAGE":"pam_unix(cron:session): session closed for user root","SYSLOG_IDENTIFIER":"CRON","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_GID":"0","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4075;b=9b6948aea6564a80a6ecb7fc6684491a;m=e392102f;t=596d620d49e35;x=d02be494a26c73d8","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_AUDIT_LOGINUID":"0","_UID":"0","_PID":"5350","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_EXE":"/usr/sbin/cron"} +{"_SYSTEMD_SLICE":"system.slice","_SOURCE_REALTIME_TIMESTAMP":"1573221901541654","_CAP_EFFECTIVE":"3fffffffff","SYSLOG_PID":"5620","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4076;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755941b;t=596d644982221;x=316c0c4c71f47f81","_GID":"0","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","_AUDIT_SESSION":"15","_UID":"0","_COMM":"cron","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_PID":"5620","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_CMDLINE":"/usr/sbin/CRON -f","_HOSTNAME":"pad","PRIORITY":"6","SYSLOG_IDENTIFIER":"CRON","_SYSTEMD_CGROUP":"/system.slice/cron.service","MESSAGE":"pam_unix(cron:session): session opened for user root by (uid=0)","_SELINUX_CONTEXT":"unconfined\n","_SYSTEMD_UNIT":"cron.service","_TRANSPORT":"syslog","SYSLOG_FACILITY":"10","__MONOTONIC_TIMESTAMP":"4418016283","_AUDIT_LOGINUID":"0","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573221901541921"} +{"_TRANSPORT":"syslog","SYSLOG_IDENTIFIER":"CRON","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","_HOSTNAME":"pad","__MONOTONIC_TIMESTAMP":"4418027363","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","PRIORITY":"6","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","SYSLOG_FACILITY":"9","_PID":"5621","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","SYSLOG_PID":"5621","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4077;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755bf63;t=596d644984d69;x=cc55cc6eb1fab69e","_GID":"0","_UID":"0","_SOURCE_REALTIME_TIMESTAMP":"1573221901544064","__REALTIME_TIMESTAMP":"1573221901553001"} +{"SYSLOG_PID":"5620","_AUDIT_LOGINUID":"0","__MONOTONIC_TIMESTAMP":"4418027830","_SYSTEMD_UNIT":"cron.service","SYSLOG_FACILITY":"10","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4078;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755c136;t=596d644984f3c;x=3b0e6ba1c3af0e15","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SELINUX_CONTEXT":"unconfined\n","_EXE":"/usr/sbin/cron","_HOSTNAME":"pad","_GID":"0","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","SYSLOG_IDENTIFIER":"CRON","MESSAGE":"pam_unix(cron:session): session closed for user root","__REALTIME_TIMESTAMP":"1573221901553468","_PID":"5620","_TRANSPORT":"syslog","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","_COMM":"cron","_CAP_EFFECTIVE":"3fffffffff","_AUDIT_SESSION":"15","_SOURCE_REALTIME_TIMESTAMP":"1573221901550207","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_SYSTEMD_SLICE":"system.slice","_SYSTEMD_CGROUP":"/system.slice/cron.service","_UID":"0","_CMDLINE":"/usr/sbin/CRON -f"} diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..95e2e45 --- /dev/null +++ b/tox.ini @@ -0,0 +1,28 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/logtracker/{env:BRANCH_NAME:} +envlist = dj111 + +[testenv] +usedevelop = true +basepython = python3 +setenv = + DJANGO_SETTINGS_MODULE=tests.settings + coverage: COVERAGE=--junit-xml=test_results.xml --cov=logtracker --cov-report xml +deps = + dj111: django>=1.11,<1.12 + coverage + psycopg2 + pytest + pytest-cov + pytest-django + tz +commands = + dj111: py.test-3 {posargs: --junitxml=test_{envname}_results.xml --cov-report xml --cov-report html --cov=logtracker -s -x tests/ --disable-warnings} +[pytest] +filterwarnings = + once:.*