start logtracker

This commit is contained in:
Christophe Siraut 2019-11-08 15:19:22 +01:00
commit 7297f1e8ef
69 changed files with 1592 additions and 0 deletions

19
.gitignore vendored Normal file
View File

@ -0,0 +1,19 @@
venv
*.ex
__pycache__
*.egg-info
*.sqlite3
*.offset
saml.*
.pybuild
build
debian/files
debian/logtracker
debian/python3-logtracker
debian/*.debhelper
debian/*.substvars
debian/debhelper-build-stamp
.coverage
coverage.xml
htmlcov/
test_dj111_results.xml

35
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,35 @@
@Library('eo-jenkins-lib@master') import eo.Utils
pipeline {
agent any
stages {
stage('Unit Tests') {
steps {
sh 'tox -rv'
}
}
stage('Packaging') {
steps {
script {
if (env.JOB_NAME == 'logtracker' && env.GIT_BRANCH == 'origin/master') {
sh 'sudo -H -u eobuilder /usr/local/bin/eobuilder -d buster logtracker'
}
}
}
}
}
post {
always {
script {
utils = new Utils()
utils.mail_notify(currentBuild, env, 'admin+logtracker@entrouvert.com')
utils.publish_coverage('coverage.xml')
utils.publish_coverage_native('index.html')
}
junit '*_results.xml'
}
success {
cleanWs()
}
}
}

6
MANIFEST.in Normal file
View File

@ -0,0 +1,6 @@
recursive-include logtracker/collection/templates *.html
recursive-include logtracker/mail/templates *.html
include Jenkinsfile
include tests/testdata.exim
include tests/testdata.journald
include tox.ini

4
README Normal file
View File

@ -0,0 +1,4 @@
Logtracker
===========
Logtracker is a django application to collect, aggregate and display log entries

5
debian/changelog vendored Normal file
View File

@ -0,0 +1,5 @@
logtracker (0.1-1) unstable; urgency=medium
* Initial release (Closes: #nnnn) <nnnn is the bug number of your ITP>
-- Christophe Siraut <csiraut@entrouvert.com> Thu, 07 Nov 2019 13:18:01 +0100

1
debian/compat vendored Normal file
View File

@ -0,0 +1 @@
10

41
debian/control vendored Normal file
View File

@ -0,0 +1,41 @@
Source: logtracker
Section: unknown
Priority: optional
Maintainer: Christophe Siraut <csiraut@entrouvert.com>
Build-Depends: debhelper (>= 10), dh-python, python3-all, python3-setuptools,
python3-django,
python3-psycopg2,
python3-tz,
python3-pytest,
python3-pytest-django
Standards-Version: 4.1.3
Homepage: https://git.entrouvert.org
X-Python3-Version: >= 3.2
#Vcs-Browser: https://salsa.debian.org/debian/logtracker
#Vcs-Git: https://salsa.debian.org/debian/logtracker.git
#Testsuite: autopkgtest-pkg-python
Package: python3-logtracker
Architecture: all
Depends: ${python3:Depends}, ${misc:Depends},
python3-psycopg2,
python3-tz
Description: Logtracker is a django application to expose email logs
This package installs the library for Python 3.
Package: logtracker
Architecture: all
Depends: ${python3:Depends}, ${misc:Depends},
postgresql,
python3-logtracker,
python3-django,
uwsgi,
uwsgi-plugin-python3
Description: Logtracker service
Recommends: nginx
Package: logtracker-agent
Architecture: all
Depends: ${python3:Depends}, ${misc:Depends},
python3-logtracker,
Description: Logtracker journald agent

38
debian/copyright vendored Normal file
View File

@ -0,0 +1,38 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: logtracker
Source: <url://example.com>
Files: *
Copyright: <years> <put author's name and email here>
<years> <likewise for another author>
License: <special license>
<Put the license of the package here indented by 1 space>
<This follows the format of Description: lines in control file>
.
<Including paragraphs>
# If you want to use GPL v2 or later for the /debian/* files use
# the following clauses, or change it to suit. Delete these two lines
Files: debian/*
Copyright: 2019 Christophe Siraut <tobald@debian.org>
License: GPL-2+
This package is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
.
This package is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
.
On Debian systems, the complete text of the GNU General
Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".
# Please also look if there are files or directories which have a
# different copyright/license attached and list them here.
# Please avoid picking licenses with terms that are more restrictive than the
# packaged work, as it may make Debian's contributions unacceptable upstream.

36
debian/logtracker-agent-generator vendored Executable file
View File

@ -0,0 +1,36 @@
#!/bin/sh
# This systemd generator creates dependency symlinks that make all logtracker
# agents listed in /etc/logtracker/agents be started/stopped/reloaded
# when logtracker-agent.service is started/stopped/reloaded.
set -eu
GENDIR="$1"
WANTDIR="$1/logtracker-agent.service.wants"
SERVICEFILE="/lib/systemd/system/logtracker-agent@.service"
AUTOSTART="all"
CONFIG_DIR=/etc/logtracker/agents
mkdir -p "$WANTDIR"
# No agent automatically started
if test "x$AUTOSTART" = "xnone" ; then
exit 0
fi
if test "x$AUTOSTART" = "xall" -o -z "$AUTOSTART" ; then
for CONFIG in `cd $CONFIG_DIR; ls *.conf 2> /dev/null`; do
NAME=${CONFIG%%.conf}
ln -s "$SERVICEFILE" "$WANTDIR/logtracker-agent@$NAME.service"
done
else
for NAME in $AUTOSTART ; do
if test -e $CONFIG_DIR/$NAME.conf ; then
ln -s "$SERVICEFILE" "$WANTDIR/logtracker-agent@$NAME.service"
fi
done
fi
exit 0

26
debian/logtracker-agent.README.Debian vendored Normal file
View File

@ -0,0 +1,26 @@
After logtracker-server has been installed, run the following commands on the server to configure agents access:
sudo -u postgres createuser logtracker_agent;
echo "ALTER USER logtracker_agent WITH PASSWORD 'new_password';" | sudo -u postgres psql
echo 'GRANT CONNECT ON DATABASE logtracker TO logtracker_agent;' | sudo -u logtracker psql
echo 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO logtracker_agent;' | sudo -u logtracker psql
echo 'GRANT INSERT ON collection_entry TO logtracker_agent;' | sudo -u logtracker psql
To configure a logtracker agent add a /etc/logtracker/agents/SOME-FILENAME.conf with:
[DEFAULT]
host = dummy-server.org
database = logtracker
user = logtracker
password = new_password
agent = journald
machine = # when targeting a container
or:
[DEFAULT]
host = dummy-server.org
database = logtracker
user = logtracker
password = new_password
agent = exim

1
debian/logtracker-agent.dirs vendored Normal file
View File

@ -0,0 +1 @@
/etc/logtracker/

1
debian/logtracker-agent.install vendored Normal file
View File

@ -0,0 +1 @@
debian/logtracker-agent-generator /lib/systemd/system-generators

15
debian/logtracker-agent.service vendored Normal file
View File

@ -0,0 +1,15 @@
# This service is actually a systemd target,
# but we are using a service since targets cannot be reloaded.
[Unit]
Description=Logtracker agent service
After=network.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/true
ExecReload=/bin/true
[Install]
WantedBy=multi-user.target

19
debian/logtracker-agent@.service vendored Normal file
View File

@ -0,0 +1,19 @@
[Unit]
Description=Logtracker agent %i
PartOf=logtracker-agent.service
ReloadPropagatedFrom=logtracker-agent.service
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
ExecStart=/usr/bin/python3 /usr/lib/python3/dist-packages/logtracker/agent/agent.py /etc/logtracker/agents/%i.conf
PIDFile=/run/logtracker/agent-%i.pid
RuntimeDirectory=logtracker
KillMode=process
ExecReload=/bin/kill -HUP $MAINPID
RestartSec=5s
Restart=on-failure
[Install]
WantedBy=multi-user.target

22
debian/logtracker-manage vendored Executable file
View File

@ -0,0 +1,22 @@
#!/bin/sh
NAME=logtracker
MANAGE=/usr/lib/$NAME/manage.py
# check user
if test x$1 = x"--forceuser"
then
shift
elif test $(id -un) != "$NAME"
then
echo "error: must use $0 with user ${NAME}"
exit 1
fi
if test $# -eq 0
then
python3 ${MANAGE} help
exit 1
fi
python3 ${MANAGE} "$@"

4
debian/logtracker.dirs vendored Normal file
View File

@ -0,0 +1,4 @@
/etc/logtracker
/usr/lib/logtracker
/var/lib/logtracker/collectstatic
/var/log/logtracker

4
debian/logtracker.install vendored Normal file
View File

@ -0,0 +1,4 @@
debian/logtracker-manage /usr/bin
debian/settings.py /etc/logtracker
debian/uwsgi.ini /etc/logtracker
debian/nginx/logtracker-example.conf /etc/nginx/sites-available

48
debian/logtracker.postinst vendored Normal file
View File

@ -0,0 +1,48 @@
#! /bin/sh
set -e
NAME="logtracker"
USER=$NAME
GROUP=$NAME
CONFIG_DIR="/etc/$NAME"
MANAGE_SCRIPT="/usr/bin/$NAME-manage"
case "$1" in
configure)
# make sure the administrative user exists
if ! getent passwd $USER >/dev/null; then
adduser --disabled-password --quiet --system \
--no-create-home --home /var/lib/$NAME \
--gecos "$NAME user" --group $USER
fi
getent group logtracker | grep adm || usermod -a -G adm logtracker
# ensure dirs ownership
chown $USER:$GROUP /var/log/$NAME
chown $USER:$GROUP /var/lib/$NAME/collectstatic
# create a secret file
SECRET_FILE=$CONFIG_DIR/secret
if [ ! -f $SECRET_FILE ]; then
echo -n "Generating Django secret..." >&2
cat /dev/urandom | tr -dc [:alnum:]-_\!\%\^:\; | head -c70 > $SECRET_FILE
chown root:$GROUP $SECRET_FILE
chmod 0440 $SECRET_FILE
fi
# Create role and database
sudo -u postgres psql -lqt | cut -d \| -f 1 | grep -q logtracker || (sudo -u postgres createuser logtracker; sudo -u postgres createdb -O logtracker logtracker;)
;;
abort-upgrade|abort-remove|abort-deconfigure)
;;
*)
echo "postinst called with unknown argument \`$1'" >&2
exit 1
;;
esac
#DEBHELPER#
exit 0

21
debian/logtracker.service vendored Normal file
View File

@ -0,0 +1,21 @@
[Unit]
Description=logtracker
After=network.target syslog.target postgresql.service
Wants=postgresql.service
[Service]
Type=notify
Environment=LANG=C.UTF-8
User=%p
Group=%p
ExecStartPre=/usr/bin/logtracker-manage migrate --noinput
ExecStartPre=/usr/bin/logtracker-manage collectstatic --noinput
ExecStart=/usr/bin/uwsgi --ini /etc/%p/uwsgi.ini
ExecReload=/bin/kill -HUP $MAINPID
KillSignal=SIGQUIT
PrivateTmp=true
Restart=on-failure
RuntimeDirectory=logtracker
[Install]
WantedBy=multi-user.target

48
debian/nginx/logtracker-example.conf vendored Normal file
View File

@ -0,0 +1,48 @@
server {
listen 443;
server_name _;
ssl on;
ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
access_log /var/log/nginx/logtracker.example.org-access.log combined;
error_log /var/log/nginx/logtracker.example.org-error.log;
location ~ ^/static/(.+)$ {
root /;
try_files /var/lib/logtracker/collectstatic/$1
=404;
}
location / {
proxy_pass http://unix:/var/run/logtracker/logtracker.sock;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-SSL on;
proxy_set_header X-Forwarded-Protocol ssl;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
server {
listen 80;
server_name _;
access_log /var/log/nginx/logtracker.example.org-access.log combined;
error_log /var/log/nginx/logtracker.example.org-error.log;
location ~ ^/static/(.+)$ {
root /;
try_files /var/lib/logtracker/collectstatic/$1
=404;
}
location / {
proxy_pass http://unix:/var/run/logtracker/logtracker.sock;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}

1
debian/python3-logtracker.dirs vendored Normal file
View File

@ -0,0 +1 @@
/usr/lib/logtracker

14
debian/rules vendored Executable file
View File

@ -0,0 +1,14 @@
#!/usr/bin/make -f
# See debhelper(7) (uncomment to enable)
# output every command that modifies files on the build system.
#export DH_VERBOSE = 1
export PYBUILD_NAME=logtracker
export PYBUILD_INSTALL_ARGS_python3=--install-scripts=/usr/lib/logtracker/
%:
dh $@ --with python3 --buildsystem=pybuild
override_dh_auto_test:
DJANGO_SETTINGS_MODULE=tests.settings pytest-3 --disable-pytest-warnings

37
debian/settings.py vendored Normal file
View File

@ -0,0 +1,37 @@
ALLOWED_HOSTS = ['logtracker.entrouvert.org']
INSTALLED_APPS += [
'mellon'
]
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'USER': 'logtracker',
'NAME': 'logtracker',
}
}
AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend',
'mellon.backends.SAMLBackend')
LOGIN_URL = 'mellon_login'
LOGOUT_URL = 'mellon_logout'
MELLON_ATTRIBUTE_MAPPING = {
'username': '{attributes[username][0]}',
'email': '{attributes[email][0]}',
'first_name': '{attributes[first_name][0]}',
'last_name': '{attributes[last_name][0]}',
}
MELLON_SUPERUSER_MAPPING = {'is_superuser': (u'true',)}
MELLON_USERNAME_TEMPLATE = '{attributes[username][0]}'
MELLON_PUBLIC_KEYS = ['/etc/logtracker/saml.crt']
MELLON_PRIVATE_KEY = '/etc/logtracker/saml.key'
MELLON_IDENTITY_PROVIDERS = [
{'METADATA': '/etc/logtracker/idp-metadata.xml',},
]

1
debian/source/format vendored Normal file
View File

@ -0,0 +1 @@
3.0 (quilt)

7
debian/uwsgi.ini vendored Normal file
View File

@ -0,0 +1,7 @@
[uwsgi]
plugin = python3
http-socket = /run/logtracker/logtracker.sock
#--wsgi-file logtracker.wsgi.py
module = logtracker.wsgi:application
chmod-socket = 666
vacuum = true

0
logtracker/__init__.py Normal file
View File

View File

44
logtracker/agent/agent.py Normal file
View File

@ -0,0 +1,44 @@
#!/usr/bin/python3
# Entrouvert 2019
import configparser
import importlib
import io
import json
import shlex
import sys
import psycopg2
import subprocess
def get_config():
configfile = sys.argv[1]
config = configparser.ConfigParser()
config.read(configfile)
return config['DEFAULT']
def tail(filename):
p = subprocess.Popen(shlex.split('tail -f %s' % filename), stdout=subprocess.PIPE)
for line in io.TextIOWrapper(p.stdout):
yield line
class Database:
def __init__(self, host, database, user, password, **kwargs):
self.conn = psycopg2.connect(host=host, database=database, user=user, password=password)
def write(self, entry):
with self.conn.cursor() as cursor:
json_data = json.dumps(entry['data'])
sql = 'insert into collection_entry (host, service, priority, timestamp, data) values (%s, %s, %s, %s, %s)'
args = (entry['host'], entry['service'], entry['priority'], entry['timestamp'], json_data)
cursor.execute(sql, args)
self.conn.commit()
if __name__ == '__main__':
config = get_config()
database = Database(**config)
agent = importlib.import_module(config['agent'])
for entry in getattr(agent, 'main')():
database.write(entry)

42
logtracker/agent/exim.py Executable file
View File

@ -0,0 +1,42 @@
#!/usr/bin/python3
# Entrouvert 2019
# Exim log parser
# See Summary of Fields in Log Lines in https://www.exim.org/exim-html-current/doc/html/spec_html/ch-log_files.html
import socket
import re
import datetime
import pytz
from django.utils import timezone
from logtracker.agent.agent import tail
host = socket.getfqdn()
paris = pytz.timezone('Europe/Paris')
patterns = {'ignore': re.compile('([\d-]+) ([\d:]+) .*(Start queue run|End queue run|daemon started|relay not permitted|Spool file is locked|Connection refused|Connection timed out|no immediate delivery|error ignored|Greylisting in action|Remote host closed connection|No route to host|SMTP error|SMTP protocol error|SMTP protocol synchronization error|SMTP command timeout|no host name found|unexpected disconnection|TLS error|log string overflowed|cancelled by timeout).*'),
'match': re.compile('(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) ([\w\d\-]+) (<=|=>|->|==|\*\*|Completed|SMTP error|Message is frozen|Frozen|Unfrozen)\s*(.*)$'),
}
def parse_date(string):
stamp = datetime.datetime.strptime(string, '%Y-%m-%d %H:%M:%S')
return timezone.make_aware(stamp, paris)
def parse_line(line):
match = re.match(patterns['match'], line)
if match:
stamp, identifier, action, raw = match.groups()
stamp = parse_date(stamp)
data = {'raw': '%s %s' % (action, raw[:511].replace("'", '')), 'identifier': identifier}
return {'host': host, 'service': 'exim', 'timestamp': stamp, 'priority': 6, 'data': data}
else:
match = re.match(patterns['ignore'], line)
if not match:
print('Failed to parse line: %s' % line)
def main():
for line in tail('/var/log/exim4/mainlog'):
match = parse_line(line)
if match:
yield match

30
logtracker/agent/journald.py Executable file
View File

@ -0,0 +1,30 @@
#!/usr/bin/python3
import io
import json
import shlex
import socket
import subprocess
import time
host = socket.getfqdn()
def read_from_journald():
p = subprocess.Popen(shlex.split('journalctl -o json -f'), stdout=subprocess.PIPE)
for line in io.TextIOWrapper(p.stdout):
yield line
def parse_line(line):
data = json.loads(line)
timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(data['__REALTIME_TIMESTAMP']) / 1000000))
return {'host': host,
'service': 'journald',
'timestamp': timestamp,
'priority': data['PRIORITY'],
'data': data}
def main():
for line in read_from_journald():
yield parse_line(line)

View File

View File

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

View File

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2019-11-12 13:58
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Entry',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('host', models.CharField(max_length=128)),
('service', models.CharField(max_length=32)),
('priority', models.IntegerField(choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7')], default=6)),
('timestamp', models.DateTimeField()),
('data', django.contrib.postgres.fields.jsonb.JSONField(blank=True)),
],
options={
'ordering': ['-timestamp', '-id'],
},
),
]

View File

@ -0,0 +1,16 @@
from django.db import models
from django.contrib.postgres.fields import JSONField
class Entry(models.Model):
host = models.CharField(max_length=128)
service = models.CharField(max_length=32)
priority = models.IntegerField(choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7')], default=6)
timestamp = models.DateTimeField()
data = JSONField(blank=True)
class Meta:
ordering = ['-timestamp', '-id']
def __str__(self):
return '%s %s %s %s' % (self.timestamp, self.host, self.service, self.data)

View File

@ -0,0 +1,12 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>{% block title %}Mailtracker{% endblock %}</title>
</head>
<body>
<div id="content">
{% block content %}{% endblock %}
</div>
</body>
</html>

View File

@ -0,0 +1,18 @@
{% extends "base.html" %}
{% block content %}
<h1>Mailtracker</h1>
<h2>Search by recipient</h2>
<form action="." method="post">{% csrf_token %}
Address:
<input type="text" name="address">
<input type="radio" name="field" value="from" checked>Sender
<input type="radio" name="field" value="to">Recipient<br>
<input type="submit" value="Search">
</form>
<a href="{% url 'senders' %}">Recent senders list</a>
{% endblock %}

View File

@ -0,0 +1,29 @@
{% extends "base.html" %}
{% block content %}
<h1>Mailtracker</h1>
<h2>Recent senders</h2>
<table>
<thead>
<tr>
<td>Sender</td>
<td>Emails Count</td>
<td>With errors</td>
<td>Pending</td>
</tr>
</thead>
<tbody>
{% for sender in object_list %}
<tr>
<td>{{ sender }}</td>
<td><a href='/api/emails?from={{ sender|urlencode }}'>{{ sender.email_count }}</a></td>
<td><a href='/api/emails?from={{ sender|urlencode }}&has_error=true'>{{ sender.error_count }}</a></td>
<td><a href='/api/emails?from={{ sender|urlencode }}&has_completed=false'>{{ sender.pending_count }}</a></td>
</tr>
{% empty %}
No data yet.
{% endfor %}
</tbody>
</table>
{% endblock %}

View File

@ -0,0 +1,28 @@
from django.contrib.auth.mixins import LoginRequiredMixin
from django.core import serializers
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.urls import reverse
from django.views.generic import TemplateView
from django.views.generic.list import ListView
from logtracker.collection.models import Entry
class EntriesList(LoginRequiredMixin, ListView):
def get_queryset(self):
qs = Entry.objects.all()
return qs[:100]
def get(self, request, *args, **kwargs):
queryset = self.get_queryset()
response = serializers.serialize("json", queryset)
return HttpResponse(response, content_type='application/json')
class HomeView(LoginRequiredMixin, TemplateView):
template_name = 'entries/home.html'
def post(self, request, *args, **kwargs):
url = '%s?%s' % (reverse('emails'), '%s=%s' % (request.POST['field'], request.POST['address']))
return HttpResponseRedirect(url)

View File

3
logtracker/mail/admin.py Normal file
View File

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

5
logtracker/mail/apps.py Normal file
View File

@ -0,0 +1,5 @@
from django.apps import AppConfig
class CollectorConfig(AppConfig):
name = 'collector'

View File

View File

@ -0,0 +1,18 @@
import datetime
from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
from logtracker.data.models import Mail
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('-d', '--days')
def handle(self, *args, **options):
if options.get('days'):
days = options.get('days')
else:
days = 3
delay = timezone.now() - datetime.timedelta(days=days)
Mail.objects.filter(stamp__lt=delay).delete()

View File

@ -0,0 +1,25 @@
from django.core.management.base import BaseCommand, CommandError
from logtracker.data.models import Entry, Mail, Sender
from logtracker.collectors import parse
from logtracker.utils import lock
class Command(BaseCommand):
help = 'Collect entries from exim4 log'
def add_arguments(self, parser):
parser.add_argument('-l', '--logfile', default='/var/log/exim4/mainlog')
parser.add_argument('--lockfile', default='/var/lock/mainlog.lock')
def handle(self, logfile, *args, **options):
with lock(options.get('lockfile')):
new_mails = []
for stamp, identifier, action, data in parse(logfile):
mail, created = Mail.objects.get_or_create(identifier=identifier)
entry = Entry(mail=mail, action=action, data=data, stamp=stamp)
entry.save()
if created:
new_mails.append(mail)
for mail in new_mails:
mail.handle_exim_data()

View File

@ -0,0 +1,9 @@
from django.core.management.base import BaseCommand, CommandError
from logtracker.data.models import Mail
class Command(BaseCommand):
help = 'Collect entries from exim4 log'
def handle(self, *args, **options):
for mail in Mail.objects.filter(has_error=True):
print(mail)

View File

@ -0,0 +1,12 @@
from django.core.management.base import BaseCommand, CommandError
from logtracker.data.models import Mail
class Command(BaseCommand):
help = 'Collect entries from exim4 log'
def add_arguments(self, parser):
parser.add_argument('sender', type=str)
def handle(self, *args, **options):
for entry in Mail.objects.filter(sender=options['sender']):
print(entry)

View File

@ -0,0 +1,9 @@
from django.core.management.base import BaseCommand, CommandError
from logtracker.data.models import Sender
class Command(BaseCommand):
help = 'Collect entries from exim4 log'
def handle(self, *args, **options):
for sender in Sender.objects.all():
print("%s %s" % (sender, sender.emails_count))

84
logtracker/mail/models.py Normal file
View File

@ -0,0 +1,84 @@
from django.db import models
import json
# Create your models here.
TYPES = [('<=', 'Arrival'),
('=>', 'Delivery'),
('==', 'Defered'),
('**', 'Failed'),
('->', 'Additional'),
('*>', 'Suppressed'),
('Completed', 'Completed'),
('Frozen', 'Frozen'),
('SMTP error', 'SMTP error'),
]
class Address(models.Model):
email = models.EmailField(blank=True, null=True)
class Meta:
abstract = True
ordering = ['email']
def email_count(self):
return self.mail_set.all().count()
def pending_count(self):
return self.mail_set.filter(has_completed=False).count()
def error_count(self):
return self.mail_set.filter(has_error=True).count()
def __str__(self):
return self.email
class Sender(Address):
pass
class Recipient(Address):
pass
class Mail(models.Model):
identifier = models.CharField(max_length=30)
sender = models.ForeignKey(Sender, null=True)
recipients = models.ManyToManyField(Recipient)
has_error = models.BooleanField(default=False)
has_completed = models.BooleanField(default=False)
stamp = models.DateTimeField(null=True)
def __str__(self):
return '%s %s' % (self.identifier, self.sender)
def handle_exim_data(self):
for entry in self.entry_set.all():
if entry.action == '<=':
address = entry.data.split(' ')[0]
sender, _ = Sender.objects.get_or_create(email=address)
self.sender = sender
self.stamp = entry.stamp
elif entry.action in ('=>', '->'):
address, _ = Recipient.objects.get_or_create(email=entry.data.split(' ')[0])
self.recipients.add(address)
elif entry.action == 'Completed':
self.has_completed = True
elif entry.action == '==' or entry.action == '**':
self.has_error = True
self.save()
class Entry(models.Model):
mail = models.ForeignKey(Mail)
data = models.CharField(max_length=512)
action = models.CharField(max_length=30, choices=TYPES)
stamp = models.DateTimeField()
class Meta:
ordering = ['stamp', 'id']
def __str__(self):
return '%s %s %s' % (self.stamp, self.action, self.data)

View File

@ -0,0 +1,12 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>{% block title %}Mailtracker{% endblock %}</title>
</head>
<body>
<div id="content">
{% block content %}{% endblock %}
</div>
</body>
</html>

View File

@ -0,0 +1,18 @@
{% extends "base.html" %}
{% block content %}
<h1>Mailtracker</h1>
<h2>Search by recipient</h2>
<form action="." method="post">{% csrf_token %}
Address:
<input type="text" name="address">
<input type="radio" name="field" value="from" checked>Sender
<input type="radio" name="field" value="to">Recipient<br>
<input type="submit" value="Search">
</form>
<a href="{% url 'senders' %}">Recent senders list</a>
{% endblock %}

View File

@ -0,0 +1,29 @@
{% extends "base.html" %}
{% block content %}
<h1>Mailtracker</h1>
<h2>Recent senders</h2>
<table>
<thead>
<tr>
<td>Sender</td>
<td>Emails Count</td>
<td>With errors</td>
<td>Pending</td>
</tr>
</thead>
<tbody>
{% for sender in object_list %}
<tr>
<td>{{ sender }}</td>
<td><a href='/api/emails?from={{ sender|urlencode }}'>{{ sender.email_count }}</a></td>
<td><a href='/api/emails?from={{ sender|urlencode }}&has_error=true'>{{ sender.error_count }}</a></td>
<td><a href='/api/emails?from={{ sender|urlencode }}&has_completed=false'>{{ sender.pending_count }}</a></td>
</tr>
{% empty %}
No data yet.
{% endfor %}
</tbody>
</table>
{% endblock %}

48
logtracker/mail/views.py Normal file
View File

@ -0,0 +1,48 @@
from urllib.parse import unquote
from django.views.generic import TemplateView
from django.views.generic.list import ListView
from django.http import JsonResponse, HttpResponseRedirect
from django.contrib.auth.mixins import LoginRequiredMixin
from django.urls import reverse
from django.shortcuts import get_object_or_404
from logtracker.mail.models import Mail, Sender, Recipient
def check(val):
return val == 'true'
class EmailsList(LoginRequiredMixin, ListView):
def get_queryset(self):
if self.request.GET.get('from'):
sender = get_object_or_404(Sender, email=unquote(self.request.GET.get('from')))
qs = sender.mail_set.all()
elif self.request.GET.get('to'):
recipient = get_object_or_404(Recipient, email=unquote(self.request.GET.get('to')))
qs = recipient.mail_set.all()
else:
qs = Mail.objects.all()
if self.request.GET.get('has_error'):
qs = qs.filter(has_error=check(self.request.GET.get('has_error')))
if self.request.GET.get('has_completed'):
qs = qs.filter(has_completed=check(self.request.GET.get('has_completed')))
return qs[:100]
def get(self, request, *args, **kwargs):
queryset = self.get_queryset()
data = {m.identifier: [str(e) for e in m.entry_set.all()] for m in queryset}
return JsonResponse({'data': data})
class MailHome(LoginRequiredMixin, TemplateView):
template_name = 'entries/home.html'
form_class = Recipient
def post(self, request, *args, **kwargs):
url = '%s?%s' % (reverse('emails'), '%s=%s' % (request.POST['field'], request.POST['address']))
return HttpResponseRedirect(url)
class SendersList(LoginRequiredMixin, ListView):
model = Sender

127
logtracker/settings.py Normal file
View File

@ -0,0 +1,127 @@
"""
Django settings for logtracker project.
Generated by 'django-admin startproject' using Django 1.11.18.
For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'dxk)c9u(gq7@2tb)+*!=$09c=o0qnwbt^vrrmjcx-bsl&2!u7b'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'logtracker.collection',
'logtracker.mail',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'logtracker.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'logtracker.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = '/static/'
STATIC_ROOT = '/var/lib/logtracker/collectstatic'
local_settings_file = '/etc/logtracker/settings.py'
if os.path.exists(local_settings_file):
exec(open(local_settings_file).read())

18
logtracker/urls.py Normal file
View File

@ -0,0 +1,18 @@
from django.conf.urls import include, url
from django.contrib import admin
from django.conf import settings
from logtracker.collection.views import EntriesList, HomeView
from logtracker.mail.views import EmailsList, SendersList, MailHome
urlpatterns = [
url(r'^admin/', admin.site.urls),
url(r'^$', HomeView.as_view()),
url(r'^api/collection/$', EntriesList.as_view()),
url(r'^api/mail/$', EmailsList.as_view(), name='emails'),
url(r'^mail/$', MailHome.as_view(), name='mail'),
url(r'^mail/senders/$', SendersList.as_view(), name='senders'),
]
if 'mellon' in settings.INSTALLED_APPS:
urlpatterns += url(r'^accounts/mellon/', include('mellon.urls')),

15
logtracker/utils.py Normal file
View File

@ -0,0 +1,15 @@
import os
import sys
import fcntl
from contextlib import contextmanager
from io import BlockingIOError
@contextmanager
def lock(lockfile):
try:
handle = open(lockfile, 'w')
fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
yield
except BlockingIOError:
print('locked')
sys.exit(1)

16
logtracker/wsgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
WSGI config for logtracker project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "logtracker.settings")
application = get_wsgi_application()

22
manage.py Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python3
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "logtracker.settings")
try:
from django.core.management import execute_from_command_line
except ImportError:
# The above import may fail for some other reason. Ensure that the
# issue is really that Django is missing to avoid masking other
# exceptions on Python 2.
try:
import django
except ImportError:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
)
raise
execute_from_command_line(sys.argv)

134
setup.py Executable file
View File

@ -0,0 +1,134 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import subprocess
import shutil
import sys
from setuptools.command.install_lib import install_lib as _install_lib
from distutils.command.build import build as _build
from setuptools.command.sdist import sdist
from distutils.cmd import Command
from setuptools import setup, find_packages
class deb(sdist):
def run(self, *args):
os.system('git clean -fdx')
version = get_version()
sdist.run(self)
shutil.move("dist/logtracker-%s.tar.gz" % version, '../logtracker_%s.orig.tar.gz' % version)
os.system('git clean -fdx')
os.system('dpkg-source -b .')
os.system('sudo HOME=$HOME DIST=buster cowbuilder --build ../logtracker_%s-1.dsc' % version)
class eo_sdist(sdist):
def run(self):
if os.path.exists('VERSION'):
os.remove('VERSION')
version = get_version()
version_file = open('VERSION', 'w')
version_file.write(version)
version_file.close()
sdist.run(self)
if os.path.exists('VERSION'):
os.remove('VERSION')
def get_version():
'''Use the VERSION, if absent generates a version with git describe, if not
tag exists, take 0.0- and add the length of the commit log.
'''
if os.path.exists('VERSION'):
with open('VERSION', 'r') as v:
return v.read()
if os.path.exists('.git'):
p = subprocess.Popen(['git','describe','--dirty=.dirty','--match=v*'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
result = p.communicate()[0]
if p.returncode == 0:
result = result.decode('ascii').strip()[1:] # strip spaces/newlines and initial v
if '-' in result: # not a tagged version
real_number, commit_count, commit_hash = result.split('-', 2)
version = '%s.post%s+%s' % (real_number, commit_count, commit_hash)
else:
version = result
return version
else:
return '0.0.post%s' % len(
subprocess.check_output(
['git', 'rev-list', 'HEAD']).splitlines())
return '0.0'
class compile_translations(Command):
description = 'compile message catalogs to MO files via django compilemessages'
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
try:
from django.core.management import call_command
for path, dirs, files in os.walk('fargo'):
if 'locale' not in dirs:
continue
curdir = os.getcwd()
os.chdir(os.path.realpath(path))
call_command('compilemessages')
os.chdir(curdir)
except ImportError:
sys.stderr.write('!!! Please install Django >= 1.4 to build translations\n')
class build(_build):
sub_commands = [('compile_translations', None)] + _build.sub_commands
class install_lib(_install_lib):
def run(self):
self.run_command('compile_translations')
_install_lib.run(self)
setup(
name='logtracker',
version=get_version(),
description='Exim Logger',
long_description=open('README').read(),
author="Entr'ouvert",
author_email='info@entrouvert.com',
packages=find_packages(),
include_package_data=True,
scripts=('manage.py',),
url='https://dev.entrouvert.org/projects/logtracker/',
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'Environment :: Web Environment',
'Framework :: Django',
'Intended Audience :: Developers',
'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 2',
],
install_requires=[
'django>=1.11',
'psycopg2',
'django-mellon'
],
zip_safe=False,
cmdclass={
'build': build,
'compile_translations': compile_translations,
'install_lib': install_lib,
'sdist': eo_sdist,
'deb': deb,
},
)

3
tests/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

48
tests/conftest.py Normal file
View File

@ -0,0 +1,48 @@
import json
import os
import pytest
import psycopg2
from django.db import connection
from django.contrib.auth.models import User
from logtracker.agent import journald, exim
basepath = os.path.dirname(__file__)
def run_sql(sql):
conn = psycopg2.connect(database='postgres')
cur = conn.cursor()
cur.execute(sql)
conn.close()
@pytest.fixture
def journald_data(request, db, client):
with open('%s/testdata.journald' % basepath) as fh:
for line in fh.readlines():
parsed = journald.parse_line(line)
write(parsed)
def write(entry):
with connection.cursor() as c:
entry['data'] = json.dumps(entry['data'])
c.execute('''insert into collection_entry (host, service, priority, timestamp, data) values (%s, %s, %s, %s, %s)''',
(entry['host'], entry['service'], entry['priority'], entry['timestamp'], entry['data']))
@pytest.fixture
def exim_data(request, db, client):
with open('%s/testdata.exim' % basepath) as fh:
for line in fh.readlines():
parsed = exim.parse_line(line)
if parsed:
write(parsed)
@pytest.fixture
def auth(db, client):
User.objects.create_user(username='john', password='Doe')
client.login(username='john', password='Doe')

123
tests/settings.py Normal file
View File

@ -0,0 +1,123 @@
"""
Django settings for logtracker project.
Generated by 'django-admin startproject' using Django 1.11.18.
For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'dxk)c9u(gq7@2tb)+*!=$09c=o0qnwbt^vrrmjcx-bsl&2!u7b'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'logtracker.collection',
'logtracker.mail'
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'logtracker.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'logtracker.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = '/static/'
STATIC_ROOT = '/var/lib/logtracker/collectstatic'

View File

@ -0,0 +1,5 @@
from logtracker.collection.models import Entry
def test_import_data(journald_data):
assert len(Entry.objects.all()) == 10

20
tests/test_base.py Normal file
View File

@ -0,0 +1,20 @@
import pytest
import json
def test_auth(client, monkeypatch):
resp = client.get('/api/mail/')
assert resp.status_code == 302
def test_home_page(client, auth):
page = client.get('/')
assert page.status_code == 200
def test_entries_list(auth, client, journald_data):
page = client.get('/api/collection/')
assert page.status_code == 200
entries = json.loads(page.content.decode())
assert len(entries) == 10

17
tests/test_mail.py Normal file
View File

@ -0,0 +1,17 @@
import pytest
import json
from django.contrib.auth.models import User
def test_mail_api(client, auth):
page = client.get('/api/mail/')
assert page.status_code == 200
def test_entries_list(auth, client, exim_data):
page = client.get('/api/collection/')
assert page.status_code == 200
entries = json.loads(page.content.decode())
assert len(entries) == 23

27
tests/testdata.exim Normal file
View File

@ -0,0 +1,27 @@
2019-02-09 14:12:32 1gsSQq-0004Nr-HL <= ne-pas-repondre@chicago.fr H=my.server.net [5.135.221.10] P=esmtp S=8785 id=E1gsSQq-0001GI-EW@auquo.myserver.net
2019-02-09 14:12:32 1gsSQq-0004Nr-HL == john.guyot33@sfr.fr R=dnslookup T=remote_smtp defer (-53): retry time not reached for any host for 'sfr.fr'
2019-02-09 14:13:11 1gsSRS-0004Nu-Mk => ymelin@gmail.com R=dnslookup T=remote_smtp H=alt1.gmail-smtp-in.l.google.com [64.233.164.26] X=TLS1.2:ECDHE_RSA_CHACHA20_POLY1305:256 CV=yes DN="C=US,ST=California,L=Mountain View,O=Google LLC,CN=mx.google.com" C="250 2.0.0 OK 1549717991 u2-v6si1114219ljg.59 - gsmtp"
2019-02-09 14:13:11 1gsSRS-0004Nu-Mk Completed
2019-02-09 14:14:12 1gsSSS-0004Nz-6i <= ne-pas-repondre@departement666.fr H=my.server.net [5.135.221.10] P=esmtp S=7977 id=20190209131412.26403.28082@authentic.node1.prod.saas.myserver.net
2019-02-09 14:14:12 1gsSSS-0004Nz-6i ** dodu.jc06@hotmail.fr R=dnslookup T=remote_smtp H=eur.olc.protection.outlook.com [104.47.5.33] X=TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=yes DN="C=US,ST=Washington,L=Redmond,O=Microsoft Corporation,CN=mail.protection.outlook.com": SMTP error from remote mail server after RCPT TO:<dodu.jc06@hotmail.fr>: 550 5.5.0 Requested action not taken: mailbox unavailable.
2019-02-09 14:14:12 1gsSSS-0004O2-PV <= <> R=1gsSSS-0004Nz-6i U=Debian-exim P=local S=9550
2019-02-09 14:14:12 1gsSSS-0004Nz-6i Completed
2019-02-09 14:14:13 1gsSSS-0004O2-PV ** ne-pas-repondre@departement666.fr R=dnslookup T=remote_smtp H=mail-av5.departement666.fr [193.48.79.5]: SMTP error from remote mail server after RCPT TO:<ne-pas-repondre@departement666.fr>: 550 5.1.1 <ne-pas-repondre@departement666.fr>: Recipient address rejected: User unknown in relay recipient table
2019-02-09 14:14:13 1gsSSS-0004O2-PV Frozen (delivery error message)
2019-02-09 14:15:45 1gsSTx-0004OW-B7 <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=8433 id=20190209131545.4912.74435@authentic.myserver.net
2019-02-09 14:15:46 1gsSTx-0004OW-B7 => emmanuel@asfar.net R=dnslookup T=remote_smtp H=a.mx.asfar.net [69.168.84.3] C="250 ok 1549718146 qp 68658"
2019-02-09 14:15:46 1gsSTx-0004OW-B7 Completed
2019-02-09 14:18:25 1gsSWW-0004Ox-VJ <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=7971 id=20190209131824.8393.69975@authentic.myserver.net
2019-02-09 14:18:31 1gsSWd-0004P2-FY <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=8283 id=20190209131831.8393.49556@authentic.myserver.net
2019-02-09 14:18:32 1gsSWd-0004P2-FY SMTP error from remote mail server after end of data: 421-4.7.0 This message does not have authentication information or fails to pass\n421-4.7.0 authentication checks. To best protect our users from spam, the\n421-4.7.0 message has been blocked. Please visit\n421-4.7.0 https://support.google.com/mail/answer/81126#authentication for more\n421 4.7.0 information. q11si3575718wro.250 - gsmtp
2019-02-09 14:21:23 1gsSZP-0004PN-0x <= ne-pas-repondre+mymeaux@myserver.net H=my.server.net [5.135.221.10] P=esmtp S=6828 id=E1gsSZN-000x9T-ST@wcs.node1.prod.saas.myserver.net
2019-02-09 14:21:23 1gsSZP-0004PN-0x => a.desales@mars.planet <A.DeSales@mars.planet> R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery"
2019-02-09 14:21:23 1gsSZP-0004PN-0x -> a.obin@mars.planet <A.obin@mars.planet> R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery"
2019-02-09 14:21:23 1gsSZP-0004PN-0x -> athy.dick@mars.planet <Athy.Dick@mars.planet> R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery"
2019-02-09 14:21:23 1gsSZP-0004PN-0x -> aurelie.rihon@mars.planet <Aurelie.Rihon@mars.planet> R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery"
2019-02-09 14:21:23 1gsSZP-0004PN-0x Completed
2019-02-09 17:25:30 H=(WIN-BR8CPBUF43R) [155.94.137.16] F=<spameri@tiscali.it> rejected RCPT <spameri@tiscali.it>: relay not permitted
2019-02-18 09:49:55 1gvecd-0008Vl-Jj H=netcourier.com [68.178.213.61]: Remote host closed connection in response to initial connection
2019-02-18 09:51:47 1gveaE-0008UG-HK H=alt2.gmail-smtp-in.l.google.com [2404:6800:4003:c02::1a] Connection timed out
2019-02-18 08:06:10 1gvbfc-0000qJ-BL H=mail.ualusa.com [168.235.95.126] Connection refused
2019-02-18 08:06:10 1gvcTW-0004qQ-RR Message is frozen

10
tests/testdata.journald Normal file
View File

@ -0,0 +1,10 @@
{"_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","SYSLOG_PID":"5115","SYSLOG_TIMESTAMP":"Nov 8 14:45:01 ","_PID":"5115","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c406f;b=9b6948aea6564a80a6ecb7fc6684491a;m=bfcea5c9;t=596d5fd1133cf;x=1e677b790058a963","_UID":"0","_GID":"0","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","_TRANSPORT":"syslog","_SOURCE_REALTIME_TIMESTAMP":"1573220701505885","SYSLOG_IDENTIFIER":"CRON","__MONOTONIC_TIMESTAMP":"3217991113","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_HOSTNAME":"pad","__REALTIME_TIMESTAMP":"1573220701516751","SYSLOG_FACILITY":"9"}
{"__MONOTONIC_TIMESTAMP":"3217991316","_SYSTEMD_UNIT":"cron.service","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_UID":"0","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573220701516954","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4070;b=9b6948aea6564a80a6ecb7fc6684491a;m=bfcea694;t=596d5fd11349a;x=488c3f2177060430","_COMM":"cron","_AUDIT_LOGINUID":"0","_SYSTEMD_CGROUP":"/system.slice/cron.service","SYSLOG_FACILITY":"10","_SOURCE_REALTIME_TIMESTAMP":"1573220701511781","_GID":"0","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SYSTEMD_SLICE":"system.slice","SYSLOG_PID":"5114","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_TRANSPORT":"syslog","_CMDLINE":"/usr/sbin/CRON -f","PRIORITY":"6","_CAP_EFFECTIVE":"3fffffffff","MESSAGE":"pam_unix(cron:session): session closed for user root","_HOSTNAME":"pad","SYSLOG_IDENTIFIER":"CRON","_SELINUX_CONTEXT":"unconfined\n","_AUDIT_SESSION":"13","SYSLOG_TIMESTAMP":"Nov 8 14:45:01 ","_PID":"5114"}
{"MESSAGE":"Valid eCryptfs headers not found in file header region or xattr region, inode 7258970","_SOURCE_MONOTONIC_TIMESTAMP":"3779354493","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4071;b=9b6948aea6564a80a6ecb7fc6684491a;m=e1426d66;t=596d61e84fb6b;x=b17588d1ff1c6b03","PRIORITY":"7","SYSLOG_FACILITY":"0","_TRANSPORT":"kernel","SYSLOG_IDENTIFIER":"kernel","__MONOTONIC_TIMESTAMP":"3779226982","_HOSTNAME":"pad","__REALTIME_TIMESTAMP":"1573221262752619"}
{"__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4072;b=9b6948aea6564a80a6ecb7fc6684491a;m=e195117c;t=596d61ed79f82;x=a9d1e5633a803563","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","__MONOTONIC_TIMESTAMP":"3784642940","_TRANSPORT":"kernel","SYSLOG_IDENTIFIER":"kernel","PRIORITY":"7","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_HOSTNAME":"pad","SYSLOG_FACILITY":"0","MESSAGE":"Valid eCryptfs headers not found in file header region or xattr region, inode 10093410","__REALTIME_TIMESTAMP":"1573221268168578","_SOURCE_MONOTONIC_TIMESTAMP":"3784771755"}
{"PRIORITY":"6","SYSLOG_IDENTIFIER":"CRON","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","SYSLOG_FACILITY":"10","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_CAP_EFFECTIVE":"3fffffffff","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SELINUX_CONTEXT":"unconfined\n","_AUDIT_SESSION":"14","_SYSTEMD_SLICE":"system.slice","_AUDIT_LOGINUID":"0","_HOSTNAME":"pad","__MONOTONIC_TIMESTAMP":"3817997608","_UID":"0","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573221301523247","_PID":"5350","_TRANSPORT":"syslog","MESSAGE":"pam_unix(cron:session): session opened for user root by (uid=0)","SYSLOG_PID":"5350","_GID":"0","_SYSTEMD_UNIT":"cron.service","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_COMM":"cron","_SYSTEMD_CGROUP":"/system.slice/cron.service","_CMDLINE":"/usr/sbin/CRON -f","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4073;b=9b6948aea6564a80a6ecb7fc6684491a;m=e3920528;t=596d620d4932f;x=4d57d832d99b5cae","_SOURCE_REALTIME_TIMESTAMP":"1573221301523129"}
{"__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4074;b=9b6948aea6564a80a6ecb7fc6684491a;m=e3920819;t=596d620d4961f;x=f8784991133557ec","SYSLOG_PID":"5351","_SYSTEMD_UNIT":"cron.service","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","__REALTIME_TIMESTAMP":"1573221301523999","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_PID":"5351","_TRANSPORT":"syslog","_GID":"0","_SYSTEMD_CGROUP":"/system.slice/cron.service","_SELINUX_CONTEXT":"unconfined\n","_UID":"0","SYSLOG_FACILITY":"9","_HOSTNAME":"pad","_CMDLINE":"/usr/sbin/CRON -f","_CAP_EFFECTIVE":"3fffffffff","_EXE":"/usr/sbin/cron","_COMM":"cron","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_SOURCE_REALTIME_TIMESTAMP":"1573221301523811","SYSLOG_IDENTIFIER":"CRON","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_SYSTEMD_SLICE":"system.slice","_AUDIT_LOGINUID":"0","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","__MONOTONIC_TIMESTAMP":"3817998361","_AUDIT_SESSION":"14"}
{"__REALTIME_TIMESTAMP":"1573221301526069","SYSLOG_FACILITY":"10","_COMM":"cron","_SELINUX_CONTEXT":"unconfined\n","PRIORITY":"6","_SYSTEMD_SLICE":"system.slice","_SYSTEMD_CGROUP":"/system.slice/cron.service","_AUDIT_SESSION":"14","_HOSTNAME":"pad","_TRANSPORT":"syslog","_CMDLINE":"/usr/sbin/CRON -f","__MONOTONIC_TIMESTAMP":"3818000431","_SOURCE_REALTIME_TIMESTAMP":"1573221301526029","SYSLOG_PID":"5350","_CAP_EFFECTIVE":"3fffffffff","_SYSTEMD_UNIT":"cron.service","MESSAGE":"pam_unix(cron:session): session closed for user root","SYSLOG_IDENTIFIER":"CRON","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_GID":"0","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4075;b=9b6948aea6564a80a6ecb7fc6684491a;m=e392102f;t=596d620d49e35;x=d02be494a26c73d8","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_AUDIT_LOGINUID":"0","_UID":"0","_PID":"5350","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_EXE":"/usr/sbin/cron"}
{"_SYSTEMD_SLICE":"system.slice","_SOURCE_REALTIME_TIMESTAMP":"1573221901541654","_CAP_EFFECTIVE":"3fffffffff","SYSLOG_PID":"5620","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4076;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755941b;t=596d644982221;x=316c0c4c71f47f81","_GID":"0","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","_AUDIT_SESSION":"15","_UID":"0","_COMM":"cron","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_PID":"5620","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_CMDLINE":"/usr/sbin/CRON -f","_HOSTNAME":"pad","PRIORITY":"6","SYSLOG_IDENTIFIER":"CRON","_SYSTEMD_CGROUP":"/system.slice/cron.service","MESSAGE":"pam_unix(cron:session): session opened for user root by (uid=0)","_SELINUX_CONTEXT":"unconfined\n","_SYSTEMD_UNIT":"cron.service","_TRANSPORT":"syslog","SYSLOG_FACILITY":"10","__MONOTONIC_TIMESTAMP":"4418016283","_AUDIT_LOGINUID":"0","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573221901541921"}
{"_TRANSPORT":"syslog","SYSLOG_IDENTIFIER":"CRON","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","_HOSTNAME":"pad","__MONOTONIC_TIMESTAMP":"4418027363","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","PRIORITY":"6","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","SYSLOG_FACILITY":"9","_PID":"5621","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","SYSLOG_PID":"5621","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4077;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755bf63;t=596d644984d69;x=cc55cc6eb1fab69e","_GID":"0","_UID":"0","_SOURCE_REALTIME_TIMESTAMP":"1573221901544064","__REALTIME_TIMESTAMP":"1573221901553001"}
{"SYSLOG_PID":"5620","_AUDIT_LOGINUID":"0","__MONOTONIC_TIMESTAMP":"4418027830","_SYSTEMD_UNIT":"cron.service","SYSLOG_FACILITY":"10","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4078;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755c136;t=596d644984f3c;x=3b0e6ba1c3af0e15","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SELINUX_CONTEXT":"unconfined\n","_EXE":"/usr/sbin/cron","_HOSTNAME":"pad","_GID":"0","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","SYSLOG_IDENTIFIER":"CRON","MESSAGE":"pam_unix(cron:session): session closed for user root","__REALTIME_TIMESTAMP":"1573221901553468","_PID":"5620","_TRANSPORT":"syslog","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","_COMM":"cron","_CAP_EFFECTIVE":"3fffffffff","_AUDIT_SESSION":"15","_SOURCE_REALTIME_TIMESTAMP":"1573221901550207","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_SYSTEMD_SLICE":"system.slice","_SYSTEMD_CGROUP":"/system.slice/cron.service","_UID":"0","_CMDLINE":"/usr/sbin/CRON -f"}

28
tox.ini Normal file
View File

@ -0,0 +1,28 @@
# Tox (http://tox.testrun.org/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.
[tox]
toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/logtracker/{env:BRANCH_NAME:}
envlist = dj111
[testenv]
usedevelop = true
basepython = python3
setenv =
DJANGO_SETTINGS_MODULE=tests.settings
coverage: COVERAGE=--junit-xml=test_results.xml --cov=logtracker --cov-report xml
deps =
dj111: django>=1.11,<1.12
coverage
psycopg2
pytest
pytest-cov
pytest-django
tz
commands =
dj111: py.test-3 {posargs: --junitxml=test_{envname}_results.xml --cov-report xml --cov-report html --cov=logtracker -s -x tests/ --disable-warnings}
[pytest]
filterwarnings =
once:.*