start logtracker
This commit is contained in:
commit
7297f1e8ef
|
@ -0,0 +1,19 @@
|
||||||
|
venv
|
||||||
|
*.ex
|
||||||
|
__pycache__
|
||||||
|
*.egg-info
|
||||||
|
*.sqlite3
|
||||||
|
*.offset
|
||||||
|
saml.*
|
||||||
|
.pybuild
|
||||||
|
build
|
||||||
|
debian/files
|
||||||
|
debian/logtracker
|
||||||
|
debian/python3-logtracker
|
||||||
|
debian/*.debhelper
|
||||||
|
debian/*.substvars
|
||||||
|
debian/debhelper-build-stamp
|
||||||
|
.coverage
|
||||||
|
coverage.xml
|
||||||
|
htmlcov/
|
||||||
|
test_dj111_results.xml
|
|
@ -0,0 +1,35 @@
|
||||||
|
@Library('eo-jenkins-lib@master') import eo.Utils
|
||||||
|
|
||||||
|
pipeline {
|
||||||
|
agent any
|
||||||
|
stages {
|
||||||
|
stage('Unit Tests') {
|
||||||
|
steps {
|
||||||
|
sh 'tox -rv'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stage('Packaging') {
|
||||||
|
steps {
|
||||||
|
script {
|
||||||
|
if (env.JOB_NAME == 'logtracker' && env.GIT_BRANCH == 'origin/master') {
|
||||||
|
sh 'sudo -H -u eobuilder /usr/local/bin/eobuilder -d buster logtracker'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
post {
|
||||||
|
always {
|
||||||
|
script {
|
||||||
|
utils = new Utils()
|
||||||
|
utils.mail_notify(currentBuild, env, 'admin+logtracker@entrouvert.com')
|
||||||
|
utils.publish_coverage('coverage.xml')
|
||||||
|
utils.publish_coverage_native('index.html')
|
||||||
|
}
|
||||||
|
junit '*_results.xml'
|
||||||
|
}
|
||||||
|
success {
|
||||||
|
cleanWs()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,6 @@
|
||||||
|
recursive-include logtracker/collection/templates *.html
|
||||||
|
recursive-include logtracker/mail/templates *.html
|
||||||
|
include Jenkinsfile
|
||||||
|
include tests/testdata.exim
|
||||||
|
include tests/testdata.journald
|
||||||
|
include tox.ini
|
|
@ -0,0 +1,4 @@
|
||||||
|
Logtracker
|
||||||
|
===========
|
||||||
|
|
||||||
|
Logtracker is a django application to collect, aggregate and display log entries
|
|
@ -0,0 +1,5 @@
|
||||||
|
logtracker (0.1-1) unstable; urgency=medium
|
||||||
|
|
||||||
|
* Initial release (Closes: #nnnn) <nnnn is the bug number of your ITP>
|
||||||
|
|
||||||
|
-- Christophe Siraut <csiraut@entrouvert.com> Thu, 07 Nov 2019 13:18:01 +0100
|
|
@ -0,0 +1 @@
|
||||||
|
10
|
|
@ -0,0 +1,41 @@
|
||||||
|
Source: logtracker
|
||||||
|
Section: unknown
|
||||||
|
Priority: optional
|
||||||
|
Maintainer: Christophe Siraut <csiraut@entrouvert.com>
|
||||||
|
Build-Depends: debhelper (>= 10), dh-python, python3-all, python3-setuptools,
|
||||||
|
python3-django,
|
||||||
|
python3-psycopg2,
|
||||||
|
python3-tz,
|
||||||
|
python3-pytest,
|
||||||
|
python3-pytest-django
|
||||||
|
Standards-Version: 4.1.3
|
||||||
|
Homepage: https://git.entrouvert.org
|
||||||
|
X-Python3-Version: >= 3.2
|
||||||
|
#Vcs-Browser: https://salsa.debian.org/debian/logtracker
|
||||||
|
#Vcs-Git: https://salsa.debian.org/debian/logtracker.git
|
||||||
|
#Testsuite: autopkgtest-pkg-python
|
||||||
|
|
||||||
|
Package: python3-logtracker
|
||||||
|
Architecture: all
|
||||||
|
Depends: ${python3:Depends}, ${misc:Depends},
|
||||||
|
python3-psycopg2,
|
||||||
|
python3-tz
|
||||||
|
Description: Logtracker is a django application to expose email logs
|
||||||
|
This package installs the library for Python 3.
|
||||||
|
|
||||||
|
Package: logtracker
|
||||||
|
Architecture: all
|
||||||
|
Depends: ${python3:Depends}, ${misc:Depends},
|
||||||
|
postgresql,
|
||||||
|
python3-logtracker,
|
||||||
|
python3-django,
|
||||||
|
uwsgi,
|
||||||
|
uwsgi-plugin-python3
|
||||||
|
Description: Logtracker service
|
||||||
|
Recommends: nginx
|
||||||
|
|
||||||
|
Package: logtracker-agent
|
||||||
|
Architecture: all
|
||||||
|
Depends: ${python3:Depends}, ${misc:Depends},
|
||||||
|
python3-logtracker,
|
||||||
|
Description: Logtracker journald agent
|
|
@ -0,0 +1,38 @@
|
||||||
|
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||||
|
Upstream-Name: logtracker
|
||||||
|
Source: <url://example.com>
|
||||||
|
|
||||||
|
Files: *
|
||||||
|
Copyright: <years> <put author's name and email here>
|
||||||
|
<years> <likewise for another author>
|
||||||
|
License: <special license>
|
||||||
|
<Put the license of the package here indented by 1 space>
|
||||||
|
<This follows the format of Description: lines in control file>
|
||||||
|
.
|
||||||
|
<Including paragraphs>
|
||||||
|
|
||||||
|
# If you want to use GPL v2 or later for the /debian/* files use
|
||||||
|
# the following clauses, or change it to suit. Delete these two lines
|
||||||
|
Files: debian/*
|
||||||
|
Copyright: 2019 Christophe Siraut <tobald@debian.org>
|
||||||
|
License: GPL-2+
|
||||||
|
This package is free software; you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation; either version 2 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
.
|
||||||
|
This package is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
.
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program. If not, see <https://www.gnu.org/licenses/>
|
||||||
|
.
|
||||||
|
On Debian systems, the complete text of the GNU General
|
||||||
|
Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".
|
||||||
|
|
||||||
|
# Please also look if there are files or directories which have a
|
||||||
|
# different copyright/license attached and list them here.
|
||||||
|
# Please avoid picking licenses with terms that are more restrictive than the
|
||||||
|
# packaged work, as it may make Debian's contributions unacceptable upstream.
|
|
@ -0,0 +1,36 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# This systemd generator creates dependency symlinks that make all logtracker
|
||||||
|
# agents listed in /etc/logtracker/agents be started/stopped/reloaded
|
||||||
|
# when logtracker-agent.service is started/stopped/reloaded.
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
GENDIR="$1"
|
||||||
|
WANTDIR="$1/logtracker-agent.service.wants"
|
||||||
|
SERVICEFILE="/lib/systemd/system/logtracker-agent@.service"
|
||||||
|
AUTOSTART="all"
|
||||||
|
CONFIG_DIR=/etc/logtracker/agents
|
||||||
|
|
||||||
|
mkdir -p "$WANTDIR"
|
||||||
|
|
||||||
|
# No agent automatically started
|
||||||
|
if test "x$AUTOSTART" = "xnone" ; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if test "x$AUTOSTART" = "xall" -o -z "$AUTOSTART" ; then
|
||||||
|
for CONFIG in `cd $CONFIG_DIR; ls *.conf 2> /dev/null`; do
|
||||||
|
NAME=${CONFIG%%.conf}
|
||||||
|
ln -s "$SERVICEFILE" "$WANTDIR/logtracker-agent@$NAME.service"
|
||||||
|
done
|
||||||
|
else
|
||||||
|
for NAME in $AUTOSTART ; do
|
||||||
|
if test -e $CONFIG_DIR/$NAME.conf ; then
|
||||||
|
ln -s "$SERVICEFILE" "$WANTDIR/logtracker-agent@$NAME.service"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit 0
|
||||||
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
After logtracker-server has been installed, run the following commands on the server to configure agents access:
|
||||||
|
|
||||||
|
sudo -u postgres createuser logtracker_agent;
|
||||||
|
echo "ALTER USER logtracker_agent WITH PASSWORD 'new_password';" | sudo -u postgres psql
|
||||||
|
echo 'GRANT CONNECT ON DATABASE logtracker TO logtracker_agent;' | sudo -u logtracker psql
|
||||||
|
echo 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO logtracker_agent;' | sudo -u logtracker psql
|
||||||
|
echo 'GRANT INSERT ON collection_entry TO logtracker_agent;' | sudo -u logtracker psql
|
||||||
|
|
||||||
|
To configure a logtracker agent add a /etc/logtracker/agents/SOME-FILENAME.conf with:
|
||||||
|
|
||||||
|
[DEFAULT]
|
||||||
|
host = dummy-server.org
|
||||||
|
database = logtracker
|
||||||
|
user = logtracker
|
||||||
|
password = new_password
|
||||||
|
agent = journald
|
||||||
|
machine = # when targeting a container
|
||||||
|
|
||||||
|
or:
|
||||||
|
|
||||||
|
[DEFAULT]
|
||||||
|
host = dummy-server.org
|
||||||
|
database = logtracker
|
||||||
|
user = logtracker
|
||||||
|
password = new_password
|
||||||
|
agent = exim
|
|
@ -0,0 +1 @@
|
||||||
|
/etc/logtracker/
|
|
@ -0,0 +1 @@
|
||||||
|
debian/logtracker-agent-generator /lib/systemd/system-generators
|
|
@ -0,0 +1,15 @@
|
||||||
|
# This service is actually a systemd target,
|
||||||
|
# but we are using a service since targets cannot be reloaded.
|
||||||
|
|
||||||
|
[Unit]
|
||||||
|
Description=Logtracker agent service
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
RemainAfterExit=yes
|
||||||
|
ExecStart=/bin/true
|
||||||
|
ExecReload=/bin/true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
|
@ -0,0 +1,19 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Logtracker agent %i
|
||||||
|
PartOf=logtracker-agent.service
|
||||||
|
ReloadPropagatedFrom=logtracker-agent.service
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=notify
|
||||||
|
ExecStart=/usr/bin/python3 /usr/lib/python3/dist-packages/logtracker/agent/agent.py /etc/logtracker/agents/%i.conf
|
||||||
|
PIDFile=/run/logtracker/agent-%i.pid
|
||||||
|
RuntimeDirectory=logtracker
|
||||||
|
KillMode=process
|
||||||
|
ExecReload=/bin/kill -HUP $MAINPID
|
||||||
|
RestartSec=5s
|
||||||
|
Restart=on-failure
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
|
@ -0,0 +1,22 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
NAME=logtracker
|
||||||
|
MANAGE=/usr/lib/$NAME/manage.py
|
||||||
|
|
||||||
|
# check user
|
||||||
|
if test x$1 = x"--forceuser"
|
||||||
|
then
|
||||||
|
shift
|
||||||
|
elif test $(id -un) != "$NAME"
|
||||||
|
then
|
||||||
|
echo "error: must use $0 with user ${NAME}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if test $# -eq 0
|
||||||
|
then
|
||||||
|
python3 ${MANAGE} help
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
python3 ${MANAGE} "$@"
|
|
@ -0,0 +1,4 @@
|
||||||
|
/etc/logtracker
|
||||||
|
/usr/lib/logtracker
|
||||||
|
/var/lib/logtracker/collectstatic
|
||||||
|
/var/log/logtracker
|
|
@ -0,0 +1,4 @@
|
||||||
|
debian/logtracker-manage /usr/bin
|
||||||
|
debian/settings.py /etc/logtracker
|
||||||
|
debian/uwsgi.ini /etc/logtracker
|
||||||
|
debian/nginx/logtracker-example.conf /etc/nginx/sites-available
|
|
@ -0,0 +1,48 @@
|
||||||
|
#! /bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
NAME="logtracker"
|
||||||
|
USER=$NAME
|
||||||
|
GROUP=$NAME
|
||||||
|
CONFIG_DIR="/etc/$NAME"
|
||||||
|
MANAGE_SCRIPT="/usr/bin/$NAME-manage"
|
||||||
|
|
||||||
|
case "$1" in
|
||||||
|
configure)
|
||||||
|
|
||||||
|
# make sure the administrative user exists
|
||||||
|
if ! getent passwd $USER >/dev/null; then
|
||||||
|
adduser --disabled-password --quiet --system \
|
||||||
|
--no-create-home --home /var/lib/$NAME \
|
||||||
|
--gecos "$NAME user" --group $USER
|
||||||
|
fi
|
||||||
|
getent group logtracker | grep adm || usermod -a -G adm logtracker
|
||||||
|
# ensure dirs ownership
|
||||||
|
chown $USER:$GROUP /var/log/$NAME
|
||||||
|
chown $USER:$GROUP /var/lib/$NAME/collectstatic
|
||||||
|
# create a secret file
|
||||||
|
SECRET_FILE=$CONFIG_DIR/secret
|
||||||
|
if [ ! -f $SECRET_FILE ]; then
|
||||||
|
echo -n "Generating Django secret..." >&2
|
||||||
|
cat /dev/urandom | tr -dc [:alnum:]-_\!\%\^:\; | head -c70 > $SECRET_FILE
|
||||||
|
chown root:$GROUP $SECRET_FILE
|
||||||
|
chmod 0440 $SECRET_FILE
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create role and database
|
||||||
|
sudo -u postgres psql -lqt | cut -d \| -f 1 | grep -q logtracker || (sudo -u postgres createuser logtracker; sudo -u postgres createdb -O logtracker logtracker;)
|
||||||
|
;;
|
||||||
|
|
||||||
|
abort-upgrade|abort-remove|abort-deconfigure)
|
||||||
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
echo "postinst called with unknown argument \`$1'" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
#DEBHELPER#
|
||||||
|
|
||||||
|
exit 0
|
|
@ -0,0 +1,21 @@
|
||||||
|
[Unit]
|
||||||
|
Description=logtracker
|
||||||
|
After=network.target syslog.target postgresql.service
|
||||||
|
Wants=postgresql.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=notify
|
||||||
|
Environment=LANG=C.UTF-8
|
||||||
|
User=%p
|
||||||
|
Group=%p
|
||||||
|
ExecStartPre=/usr/bin/logtracker-manage migrate --noinput
|
||||||
|
ExecStartPre=/usr/bin/logtracker-manage collectstatic --noinput
|
||||||
|
ExecStart=/usr/bin/uwsgi --ini /etc/%p/uwsgi.ini
|
||||||
|
ExecReload=/bin/kill -HUP $MAINPID
|
||||||
|
KillSignal=SIGQUIT
|
||||||
|
PrivateTmp=true
|
||||||
|
Restart=on-failure
|
||||||
|
RuntimeDirectory=logtracker
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
|
@ -0,0 +1,48 @@
|
||||||
|
server {
|
||||||
|
listen 443;
|
||||||
|
server_name _;
|
||||||
|
|
||||||
|
ssl on;
|
||||||
|
ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
|
||||||
|
ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
|
||||||
|
|
||||||
|
access_log /var/log/nginx/logtracker.example.org-access.log combined;
|
||||||
|
error_log /var/log/nginx/logtracker.example.org-error.log;
|
||||||
|
|
||||||
|
location ~ ^/static/(.+)$ {
|
||||||
|
root /;
|
||||||
|
try_files /var/lib/logtracker/collectstatic/$1
|
||||||
|
=404;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://unix:/var/run/logtracker/logtracker.sock;
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Forwarded-SSL on;
|
||||||
|
proxy_set_header X-Forwarded-Protocol ssl;
|
||||||
|
proxy_set_header X-Forwarded-Proto https;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name _;
|
||||||
|
|
||||||
|
access_log /var/log/nginx/logtracker.example.org-access.log combined;
|
||||||
|
error_log /var/log/nginx/logtracker.example.org-error.log;
|
||||||
|
|
||||||
|
location ~ ^/static/(.+)$ {
|
||||||
|
root /;
|
||||||
|
try_files /var/lib/logtracker/collectstatic/$1
|
||||||
|
=404;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://unix:/var/run/logtracker/logtracker.sock;
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
/usr/lib/logtracker
|
|
@ -0,0 +1,14 @@
|
||||||
|
#!/usr/bin/make -f
|
||||||
|
# See debhelper(7) (uncomment to enable)
|
||||||
|
# output every command that modifies files on the build system.
|
||||||
|
#export DH_VERBOSE = 1
|
||||||
|
|
||||||
|
export PYBUILD_NAME=logtracker
|
||||||
|
export PYBUILD_INSTALL_ARGS_python3=--install-scripts=/usr/lib/logtracker/
|
||||||
|
|
||||||
|
%:
|
||||||
|
dh $@ --with python3 --buildsystem=pybuild
|
||||||
|
|
||||||
|
|
||||||
|
override_dh_auto_test:
|
||||||
|
DJANGO_SETTINGS_MODULE=tests.settings pytest-3 --disable-pytest-warnings
|
|
@ -0,0 +1,37 @@
|
||||||
|
ALLOWED_HOSTS = ['logtracker.entrouvert.org']
|
||||||
|
|
||||||
|
INSTALLED_APPS += [
|
||||||
|
'mellon'
|
||||||
|
]
|
||||||
|
|
||||||
|
DATABASES = {
|
||||||
|
'default': {
|
||||||
|
'ENGINE': 'django.db.backends.postgresql',
|
||||||
|
'USER': 'logtracker',
|
||||||
|
'NAME': 'logtracker',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend',
|
||||||
|
'mellon.backends.SAMLBackend')
|
||||||
|
|
||||||
|
LOGIN_URL = 'mellon_login'
|
||||||
|
LOGOUT_URL = 'mellon_logout'
|
||||||
|
|
||||||
|
MELLON_ATTRIBUTE_MAPPING = {
|
||||||
|
'username': '{attributes[username][0]}',
|
||||||
|
'email': '{attributes[email][0]}',
|
||||||
|
'first_name': '{attributes[first_name][0]}',
|
||||||
|
'last_name': '{attributes[last_name][0]}',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
MELLON_SUPERUSER_MAPPING = {'is_superuser': (u'true',)}
|
||||||
|
MELLON_USERNAME_TEMPLATE = '{attributes[username][0]}'
|
||||||
|
|
||||||
|
MELLON_PUBLIC_KEYS = ['/etc/logtracker/saml.crt']
|
||||||
|
MELLON_PRIVATE_KEY = '/etc/logtracker/saml.key'
|
||||||
|
MELLON_IDENTITY_PROVIDERS = [
|
||||||
|
{'METADATA': '/etc/logtracker/idp-metadata.xml',},
|
||||||
|
]
|
|
@ -0,0 +1 @@
|
||||||
|
3.0 (quilt)
|
|
@ -0,0 +1,7 @@
|
||||||
|
[uwsgi]
|
||||||
|
plugin = python3
|
||||||
|
http-socket = /run/logtracker/logtracker.sock
|
||||||
|
#--wsgi-file logtracker.wsgi.py
|
||||||
|
module = logtracker.wsgi:application
|
||||||
|
chmod-socket = 666
|
||||||
|
vacuum = true
|
|
@ -0,0 +1,44 @@
|
||||||
|
#!/usr/bin/python3
|
||||||
|
# Entrouvert 2019
|
||||||
|
import configparser
|
||||||
|
import importlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import shlex
|
||||||
|
import sys
|
||||||
|
import psycopg2
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
def get_config():
|
||||||
|
configfile = sys.argv[1]
|
||||||
|
config = configparser.ConfigParser()
|
||||||
|
config.read(configfile)
|
||||||
|
return config['DEFAULT']
|
||||||
|
|
||||||
|
|
||||||
|
def tail(filename):
|
||||||
|
p = subprocess.Popen(shlex.split('tail -f %s' % filename), stdout=subprocess.PIPE)
|
||||||
|
for line in io.TextIOWrapper(p.stdout):
|
||||||
|
yield line
|
||||||
|
|
||||||
|
|
||||||
|
class Database:
|
||||||
|
def __init__(self, host, database, user, password, **kwargs):
|
||||||
|
self.conn = psycopg2.connect(host=host, database=database, user=user, password=password)
|
||||||
|
|
||||||
|
def write(self, entry):
|
||||||
|
with self.conn.cursor() as cursor:
|
||||||
|
json_data = json.dumps(entry['data'])
|
||||||
|
sql = 'insert into collection_entry (host, service, priority, timestamp, data) values (%s, %s, %s, %s, %s)'
|
||||||
|
args = (entry['host'], entry['service'], entry['priority'], entry['timestamp'], json_data)
|
||||||
|
cursor.execute(sql, args)
|
||||||
|
self.conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
config = get_config()
|
||||||
|
database = Database(**config)
|
||||||
|
agent = importlib.import_module(config['agent'])
|
||||||
|
for entry in getattr(agent, 'main')():
|
||||||
|
database.write(entry)
|
|
@ -0,0 +1,42 @@
|
||||||
|
#!/usr/bin/python3
|
||||||
|
# Entrouvert 2019
|
||||||
|
# Exim log parser
|
||||||
|
# See Summary of Fields in Log Lines in https://www.exim.org/exim-html-current/doc/html/spec_html/ch-log_files.html
|
||||||
|
import socket
|
||||||
|
import re
|
||||||
|
import datetime
|
||||||
|
import pytz
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from logtracker.agent.agent import tail
|
||||||
|
|
||||||
|
host = socket.getfqdn()
|
||||||
|
paris = pytz.timezone('Europe/Paris')
|
||||||
|
patterns = {'ignore': re.compile('([\d-]+) ([\d:]+) .*(Start queue run|End queue run|daemon started|relay not permitted|Spool file is locked|Connection refused|Connection timed out|no immediate delivery|error ignored|Greylisting in action|Remote host closed connection|No route to host|SMTP error|SMTP protocol error|SMTP protocol synchronization error|SMTP command timeout|no host name found|unexpected disconnection|TLS error|log string overflowed|cancelled by timeout).*'),
|
||||||
|
'match': re.compile('(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) ([\w\d\-]+) (<=|=>|->|==|\*\*|Completed|SMTP error|Message is frozen|Frozen|Unfrozen)\s*(.*)$'),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_date(string):
|
||||||
|
stamp = datetime.datetime.strptime(string, '%Y-%m-%d %H:%M:%S')
|
||||||
|
return timezone.make_aware(stamp, paris)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_line(line):
|
||||||
|
match = re.match(patterns['match'], line)
|
||||||
|
if match:
|
||||||
|
stamp, identifier, action, raw = match.groups()
|
||||||
|
stamp = parse_date(stamp)
|
||||||
|
data = {'raw': '%s %s' % (action, raw[:511].replace("'", '')), 'identifier': identifier}
|
||||||
|
return {'host': host, 'service': 'exim', 'timestamp': stamp, 'priority': 6, 'data': data}
|
||||||
|
else:
|
||||||
|
match = re.match(patterns['ignore'], line)
|
||||||
|
if not match:
|
||||||
|
print('Failed to parse line: %s' % line)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
for line in tail('/var/log/exim4/mainlog'):
|
||||||
|
match = parse_line(line)
|
||||||
|
if match:
|
||||||
|
yield match
|
|
@ -0,0 +1,30 @@
|
||||||
|
#!/usr/bin/python3
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import shlex
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
|
||||||
|
host = socket.getfqdn()
|
||||||
|
|
||||||
|
|
||||||
|
def read_from_journald():
|
||||||
|
p = subprocess.Popen(shlex.split('journalctl -o json -f'), stdout=subprocess.PIPE)
|
||||||
|
for line in io.TextIOWrapper(p.stdout):
|
||||||
|
yield line
|
||||||
|
|
||||||
|
|
||||||
|
def parse_line(line):
|
||||||
|
data = json.loads(line)
|
||||||
|
timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(data['__REALTIME_TIMESTAMP']) / 1000000))
|
||||||
|
return {'host': host,
|
||||||
|
'service': 'journald',
|
||||||
|
'timestamp': timestamp,
|
||||||
|
'priority': data['PRIORITY'],
|
||||||
|
'data': data}
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
for line in read_from_journald():
|
||||||
|
yield parse_line(line)
|
|
@ -0,0 +1,3 @@
|
||||||
|
from django.contrib import admin
|
||||||
|
|
||||||
|
# Register your models here.
|
|
@ -0,0 +1,31 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Generated by Django 1.11.23 on 2019-11-12 13:58
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import django.contrib.postgres.fields.jsonb
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
initial = True
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='Entry',
|
||||||
|
fields=[
|
||||||
|
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||||
|
('host', models.CharField(max_length=128)),
|
||||||
|
('service', models.CharField(max_length=32)),
|
||||||
|
('priority', models.IntegerField(choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7')], default=6)),
|
||||||
|
('timestamp', models.DateTimeField()),
|
||||||
|
('data', django.contrib.postgres.fields.jsonb.JSONField(blank=True)),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'ordering': ['-timestamp', '-id'],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
|
@ -0,0 +1,16 @@
|
||||||
|
from django.db import models
|
||||||
|
from django.contrib.postgres.fields import JSONField
|
||||||
|
|
||||||
|
|
||||||
|
class Entry(models.Model):
|
||||||
|
host = models.CharField(max_length=128)
|
||||||
|
service = models.CharField(max_length=32)
|
||||||
|
priority = models.IntegerField(choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7')], default=6)
|
||||||
|
timestamp = models.DateTimeField()
|
||||||
|
data = JSONField(blank=True)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
ordering = ['-timestamp', '-id']
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return '%s %s %s %s' % (self.timestamp, self.host, self.service, self.data)
|
|
@ -0,0 +1,12 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<title>{% block title %}Mailtracker{% endblock %}</title>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<div id="content">
|
||||||
|
{% block content %}{% endblock %}
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
|
@ -0,0 +1,18 @@
|
||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<h1>Mailtracker</h1>
|
||||||
|
|
||||||
|
<h2>Search by recipient</h2>
|
||||||
|
|
||||||
|
<form action="." method="post">{% csrf_token %}
|
||||||
|
Address:
|
||||||
|
<input type="text" name="address">
|
||||||
|
<input type="radio" name="field" value="from" checked>Sender
|
||||||
|
<input type="radio" name="field" value="to">Recipient<br>
|
||||||
|
<input type="submit" value="Search">
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<a href="{% url 'senders' %}">Recent senders list</a>
|
||||||
|
|
||||||
|
{% endblock %}
|
|
@ -0,0 +1,29 @@
|
||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<h1>Mailtracker</h1>
|
||||||
|
|
||||||
|
<h2>Recent senders</h2>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<td>Sender</td>
|
||||||
|
<td>Emails Count</td>
|
||||||
|
<td>With errors</td>
|
||||||
|
<td>Pending</td>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for sender in object_list %}
|
||||||
|
<tr>
|
||||||
|
<td>{{ sender }}</td>
|
||||||
|
<td><a href='/api/emails?from={{ sender|urlencode }}'>{{ sender.email_count }}</a></td>
|
||||||
|
<td><a href='/api/emails?from={{ sender|urlencode }}&has_error=true'>{{ sender.error_count }}</a></td>
|
||||||
|
<td><a href='/api/emails?from={{ sender|urlencode }}&has_completed=false'>{{ sender.pending_count }}</a></td>
|
||||||
|
</tr>
|
||||||
|
{% empty %}
|
||||||
|
No data yet.
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{% endblock %}
|
|
@ -0,0 +1,28 @@
|
||||||
|
from django.contrib.auth.mixins import LoginRequiredMixin
|
||||||
|
from django.core import serializers
|
||||||
|
from django.http import HttpResponse, HttpResponseRedirect
|
||||||
|
from django.shortcuts import get_object_or_404
|
||||||
|
from django.urls import reverse
|
||||||
|
from django.views.generic import TemplateView
|
||||||
|
from django.views.generic.list import ListView
|
||||||
|
|
||||||
|
from logtracker.collection.models import Entry
|
||||||
|
|
||||||
|
|
||||||
|
class EntriesList(LoginRequiredMixin, ListView):
|
||||||
|
def get_queryset(self):
|
||||||
|
qs = Entry.objects.all()
|
||||||
|
return qs[:100]
|
||||||
|
|
||||||
|
def get(self, request, *args, **kwargs):
|
||||||
|
queryset = self.get_queryset()
|
||||||
|
response = serializers.serialize("json", queryset)
|
||||||
|
return HttpResponse(response, content_type='application/json')
|
||||||
|
|
||||||
|
|
||||||
|
class HomeView(LoginRequiredMixin, TemplateView):
|
||||||
|
template_name = 'entries/home.html'
|
||||||
|
|
||||||
|
def post(self, request, *args, **kwargs):
|
||||||
|
url = '%s?%s' % (reverse('emails'), '%s=%s' % (request.POST['field'], request.POST['address']))
|
||||||
|
return HttpResponseRedirect(url)
|
|
@ -0,0 +1,3 @@
|
||||||
|
from django.contrib import admin
|
||||||
|
|
||||||
|
# Register your models here.
|
|
@ -0,0 +1,5 @@
|
||||||
|
from django.apps import AppConfig
|
||||||
|
|
||||||
|
|
||||||
|
class CollectorConfig(AppConfig):
|
||||||
|
name = 'collector'
|
|
@ -0,0 +1,18 @@
|
||||||
|
import datetime
|
||||||
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from logtracker.data.models import Mail
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
|
||||||
|
def add_arguments(self, parser):
|
||||||
|
parser.add_argument('-d', '--days')
|
||||||
|
|
||||||
|
def handle(self, *args, **options):
|
||||||
|
if options.get('days'):
|
||||||
|
days = options.get('days')
|
||||||
|
else:
|
||||||
|
days = 3
|
||||||
|
delay = timezone.now() - datetime.timedelta(days=days)
|
||||||
|
Mail.objects.filter(stamp__lt=delay).delete()
|
|
@ -0,0 +1,25 @@
|
||||||
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
from logtracker.data.models import Entry, Mail, Sender
|
||||||
|
from logtracker.collectors import parse
|
||||||
|
from logtracker.utils import lock
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = 'Collect entries from exim4 log'
|
||||||
|
|
||||||
|
def add_arguments(self, parser):
|
||||||
|
parser.add_argument('-l', '--logfile', default='/var/log/exim4/mainlog')
|
||||||
|
parser.add_argument('--lockfile', default='/var/lock/mainlog.lock')
|
||||||
|
|
||||||
|
def handle(self, logfile, *args, **options):
|
||||||
|
with lock(options.get('lockfile')):
|
||||||
|
new_mails = []
|
||||||
|
for stamp, identifier, action, data in parse(logfile):
|
||||||
|
mail, created = Mail.objects.get_or_create(identifier=identifier)
|
||||||
|
entry = Entry(mail=mail, action=action, data=data, stamp=stamp)
|
||||||
|
entry.save()
|
||||||
|
if created:
|
||||||
|
new_mails.append(mail)
|
||||||
|
|
||||||
|
for mail in new_mails:
|
||||||
|
mail.handle_exim_data()
|
|
@ -0,0 +1,9 @@
|
||||||
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
from logtracker.data.models import Mail
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = 'Collect entries from exim4 log'
|
||||||
|
|
||||||
|
def handle(self, *args, **options):
|
||||||
|
for mail in Mail.objects.filter(has_error=True):
|
||||||
|
print(mail)
|
|
@ -0,0 +1,12 @@
|
||||||
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
from logtracker.data.models import Mail
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = 'Collect entries from exim4 log'
|
||||||
|
|
||||||
|
def add_arguments(self, parser):
|
||||||
|
parser.add_argument('sender', type=str)
|
||||||
|
|
||||||
|
def handle(self, *args, **options):
|
||||||
|
for entry in Mail.objects.filter(sender=options['sender']):
|
||||||
|
print(entry)
|
|
@ -0,0 +1,9 @@
|
||||||
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
from logtracker.data.models import Sender
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = 'Collect entries from exim4 log'
|
||||||
|
|
||||||
|
def handle(self, *args, **options):
|
||||||
|
for sender in Sender.objects.all():
|
||||||
|
print("%s %s" % (sender, sender.emails_count))
|
|
@ -0,0 +1,84 @@
|
||||||
|
from django.db import models
|
||||||
|
import json
|
||||||
|
|
||||||
|
# Create your models here.
|
||||||
|
|
||||||
|
TYPES = [('<=', 'Arrival'),
|
||||||
|
('=>', 'Delivery'),
|
||||||
|
('==', 'Defered'),
|
||||||
|
('**', 'Failed'),
|
||||||
|
('->', 'Additional'),
|
||||||
|
('*>', 'Suppressed'),
|
||||||
|
('Completed', 'Completed'),
|
||||||
|
('Frozen', 'Frozen'),
|
||||||
|
('SMTP error', 'SMTP error'),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class Address(models.Model):
|
||||||
|
email = models.EmailField(blank=True, null=True)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
abstract = True
|
||||||
|
ordering = ['email']
|
||||||
|
|
||||||
|
def email_count(self):
|
||||||
|
return self.mail_set.all().count()
|
||||||
|
|
||||||
|
def pending_count(self):
|
||||||
|
return self.mail_set.filter(has_completed=False).count()
|
||||||
|
|
||||||
|
def error_count(self):
|
||||||
|
return self.mail_set.filter(has_error=True).count()
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return self.email
|
||||||
|
|
||||||
|
|
||||||
|
class Sender(Address):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Recipient(Address):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Mail(models.Model):
|
||||||
|
identifier = models.CharField(max_length=30)
|
||||||
|
sender = models.ForeignKey(Sender, null=True)
|
||||||
|
recipients = models.ManyToManyField(Recipient)
|
||||||
|
has_error = models.BooleanField(default=False)
|
||||||
|
has_completed = models.BooleanField(default=False)
|
||||||
|
stamp = models.DateTimeField(null=True)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return '%s %s' % (self.identifier, self.sender)
|
||||||
|
|
||||||
|
def handle_exim_data(self):
|
||||||
|
for entry in self.entry_set.all():
|
||||||
|
if entry.action == '<=':
|
||||||
|
address = entry.data.split(' ')[0]
|
||||||
|
sender, _ = Sender.objects.get_or_create(email=address)
|
||||||
|
self.sender = sender
|
||||||
|
self.stamp = entry.stamp
|
||||||
|
elif entry.action in ('=>', '->'):
|
||||||
|
address, _ = Recipient.objects.get_or_create(email=entry.data.split(' ')[0])
|
||||||
|
self.recipients.add(address)
|
||||||
|
elif entry.action == 'Completed':
|
||||||
|
self.has_completed = True
|
||||||
|
elif entry.action == '==' or entry.action == '**':
|
||||||
|
self.has_error = True
|
||||||
|
self.save()
|
||||||
|
|
||||||
|
|
||||||
|
class Entry(models.Model):
|
||||||
|
mail = models.ForeignKey(Mail)
|
||||||
|
data = models.CharField(max_length=512)
|
||||||
|
action = models.CharField(max_length=30, choices=TYPES)
|
||||||
|
stamp = models.DateTimeField()
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
ordering = ['stamp', 'id']
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return '%s %s %s' % (self.stamp, self.action, self.data)
|
|
@ -0,0 +1,12 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<title>{% block title %}Mailtracker{% endblock %}</title>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<div id="content">
|
||||||
|
{% block content %}{% endblock %}
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
|
@ -0,0 +1,18 @@
|
||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<h1>Mailtracker</h1>
|
||||||
|
|
||||||
|
<h2>Search by recipient</h2>
|
||||||
|
|
||||||
|
<form action="." method="post">{% csrf_token %}
|
||||||
|
Address:
|
||||||
|
<input type="text" name="address">
|
||||||
|
<input type="radio" name="field" value="from" checked>Sender
|
||||||
|
<input type="radio" name="field" value="to">Recipient<br>
|
||||||
|
<input type="submit" value="Search">
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<a href="{% url 'senders' %}">Recent senders list</a>
|
||||||
|
|
||||||
|
{% endblock %}
|
|
@ -0,0 +1,29 @@
|
||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<h1>Mailtracker</h1>
|
||||||
|
|
||||||
|
<h2>Recent senders</h2>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<td>Sender</td>
|
||||||
|
<td>Emails Count</td>
|
||||||
|
<td>With errors</td>
|
||||||
|
<td>Pending</td>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for sender in object_list %}
|
||||||
|
<tr>
|
||||||
|
<td>{{ sender }}</td>
|
||||||
|
<td><a href='/api/emails?from={{ sender|urlencode }}'>{{ sender.email_count }}</a></td>
|
||||||
|
<td><a href='/api/emails?from={{ sender|urlencode }}&has_error=true'>{{ sender.error_count }}</a></td>
|
||||||
|
<td><a href='/api/emails?from={{ sender|urlencode }}&has_completed=false'>{{ sender.pending_count }}</a></td>
|
||||||
|
</tr>
|
||||||
|
{% empty %}
|
||||||
|
No data yet.
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{% endblock %}
|
|
@ -0,0 +1,48 @@
|
||||||
|
from urllib.parse import unquote
|
||||||
|
from django.views.generic import TemplateView
|
||||||
|
from django.views.generic.list import ListView
|
||||||
|
from django.http import JsonResponse, HttpResponseRedirect
|
||||||
|
from django.contrib.auth.mixins import LoginRequiredMixin
|
||||||
|
from django.urls import reverse
|
||||||
|
from django.shortcuts import get_object_or_404
|
||||||
|
from logtracker.mail.models import Mail, Sender, Recipient
|
||||||
|
|
||||||
|
|
||||||
|
def check(val):
|
||||||
|
return val == 'true'
|
||||||
|
|
||||||
|
|
||||||
|
class EmailsList(LoginRequiredMixin, ListView):
|
||||||
|
def get_queryset(self):
|
||||||
|
if self.request.GET.get('from'):
|
||||||
|
sender = get_object_or_404(Sender, email=unquote(self.request.GET.get('from')))
|
||||||
|
qs = sender.mail_set.all()
|
||||||
|
elif self.request.GET.get('to'):
|
||||||
|
recipient = get_object_or_404(Recipient, email=unquote(self.request.GET.get('to')))
|
||||||
|
qs = recipient.mail_set.all()
|
||||||
|
else:
|
||||||
|
qs = Mail.objects.all()
|
||||||
|
if self.request.GET.get('has_error'):
|
||||||
|
qs = qs.filter(has_error=check(self.request.GET.get('has_error')))
|
||||||
|
if self.request.GET.get('has_completed'):
|
||||||
|
qs = qs.filter(has_completed=check(self.request.GET.get('has_completed')))
|
||||||
|
return qs[:100]
|
||||||
|
|
||||||
|
def get(self, request, *args, **kwargs):
|
||||||
|
queryset = self.get_queryset()
|
||||||
|
data = {m.identifier: [str(e) for e in m.entry_set.all()] for m in queryset}
|
||||||
|
return JsonResponse({'data': data})
|
||||||
|
|
||||||
|
|
||||||
|
class MailHome(LoginRequiredMixin, TemplateView):
|
||||||
|
template_name = 'entries/home.html'
|
||||||
|
form_class = Recipient
|
||||||
|
|
||||||
|
def post(self, request, *args, **kwargs):
|
||||||
|
url = '%s?%s' % (reverse('emails'), '%s=%s' % (request.POST['field'], request.POST['address']))
|
||||||
|
return HttpResponseRedirect(url)
|
||||||
|
|
||||||
|
|
||||||
|
class SendersList(LoginRequiredMixin, ListView):
|
||||||
|
|
||||||
|
model = Sender
|
|
@ -0,0 +1,127 @@
|
||||||
|
"""
|
||||||
|
Django settings for logtracker project.
|
||||||
|
|
||||||
|
Generated by 'django-admin startproject' using Django 1.11.18.
|
||||||
|
|
||||||
|
For more information on this file, see
|
||||||
|
https://docs.djangoproject.com/en/1.11/topics/settings/
|
||||||
|
|
||||||
|
For the full list of settings and their values, see
|
||||||
|
https://docs.djangoproject.com/en/1.11/ref/settings/
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||||
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
|
||||||
|
# Quick-start development settings - unsuitable for production
|
||||||
|
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
|
||||||
|
|
||||||
|
# SECURITY WARNING: keep the secret key used in production secret!
|
||||||
|
SECRET_KEY = 'dxk)c9u(gq7@2tb)+*!=$09c=o0qnwbt^vrrmjcx-bsl&2!u7b'
|
||||||
|
|
||||||
|
# SECURITY WARNING: don't run with debug turned on in production!
|
||||||
|
DEBUG = True
|
||||||
|
|
||||||
|
ALLOWED_HOSTS = []
|
||||||
|
|
||||||
|
|
||||||
|
# Application definition
|
||||||
|
|
||||||
|
INSTALLED_APPS = [
|
||||||
|
'django.contrib.admin',
|
||||||
|
'django.contrib.auth',
|
||||||
|
'django.contrib.contenttypes',
|
||||||
|
'django.contrib.sessions',
|
||||||
|
'django.contrib.messages',
|
||||||
|
'django.contrib.staticfiles',
|
||||||
|
'logtracker.collection',
|
||||||
|
'logtracker.mail',
|
||||||
|
]
|
||||||
|
|
||||||
|
MIDDLEWARE = [
|
||||||
|
'django.middleware.security.SecurityMiddleware',
|
||||||
|
'django.contrib.sessions.middleware.SessionMiddleware',
|
||||||
|
'django.middleware.common.CommonMiddleware',
|
||||||
|
'django.middleware.csrf.CsrfViewMiddleware',
|
||||||
|
'django.contrib.auth.middleware.AuthenticationMiddleware',
|
||||||
|
'django.contrib.messages.middleware.MessageMiddleware',
|
||||||
|
'django.middleware.clickjacking.XFrameOptionsMiddleware',
|
||||||
|
]
|
||||||
|
|
||||||
|
ROOT_URLCONF = 'logtracker.urls'
|
||||||
|
|
||||||
|
TEMPLATES = [
|
||||||
|
{
|
||||||
|
'BACKEND': 'django.template.backends.django.DjangoTemplates',
|
||||||
|
'DIRS': [],
|
||||||
|
'APP_DIRS': True,
|
||||||
|
'OPTIONS': {
|
||||||
|
'context_processors': [
|
||||||
|
'django.template.context_processors.debug',
|
||||||
|
'django.template.context_processors.request',
|
||||||
|
'django.contrib.auth.context_processors.auth',
|
||||||
|
'django.contrib.messages.context_processors.messages',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
WSGI_APPLICATION = 'logtracker.wsgi.application'
|
||||||
|
|
||||||
|
|
||||||
|
# Database
|
||||||
|
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
|
||||||
|
|
||||||
|
DATABASES = {
|
||||||
|
'default': {
|
||||||
|
'ENGINE': 'django.db.backends.sqlite3',
|
||||||
|
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Password validation
|
||||||
|
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
|
||||||
|
|
||||||
|
AUTH_PASSWORD_VALIDATORS = [
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# Internationalization
|
||||||
|
# https://docs.djangoproject.com/en/1.11/topics/i18n/
|
||||||
|
|
||||||
|
LANGUAGE_CODE = 'en-us'
|
||||||
|
|
||||||
|
TIME_ZONE = 'UTC'
|
||||||
|
|
||||||
|
USE_I18N = True
|
||||||
|
|
||||||
|
USE_L10N = True
|
||||||
|
|
||||||
|
USE_TZ = True
|
||||||
|
|
||||||
|
|
||||||
|
# Static files (CSS, JavaScript, Images)
|
||||||
|
# https://docs.djangoproject.com/en/1.11/howto/static-files/
|
||||||
|
|
||||||
|
STATIC_URL = '/static/'
|
||||||
|
STATIC_ROOT = '/var/lib/logtracker/collectstatic'
|
||||||
|
|
||||||
|
local_settings_file = '/etc/logtracker/settings.py'
|
||||||
|
if os.path.exists(local_settings_file):
|
||||||
|
exec(open(local_settings_file).read())
|
|
@ -0,0 +1,18 @@
|
||||||
|
from django.conf.urls import include, url
|
||||||
|
from django.contrib import admin
|
||||||
|
from django.conf import settings
|
||||||
|
|
||||||
|
from logtracker.collection.views import EntriesList, HomeView
|
||||||
|
from logtracker.mail.views import EmailsList, SendersList, MailHome
|
||||||
|
|
||||||
|
urlpatterns = [
|
||||||
|
url(r'^admin/', admin.site.urls),
|
||||||
|
url(r'^$', HomeView.as_view()),
|
||||||
|
url(r'^api/collection/$', EntriesList.as_view()),
|
||||||
|
url(r'^api/mail/$', EmailsList.as_view(), name='emails'),
|
||||||
|
url(r'^mail/$', MailHome.as_view(), name='mail'),
|
||||||
|
url(r'^mail/senders/$', SendersList.as_view(), name='senders'),
|
||||||
|
]
|
||||||
|
|
||||||
|
if 'mellon' in settings.INSTALLED_APPS:
|
||||||
|
urlpatterns += url(r'^accounts/mellon/', include('mellon.urls')),
|
|
@ -0,0 +1,15 @@
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import fcntl
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from io import BlockingIOError
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def lock(lockfile):
|
||||||
|
try:
|
||||||
|
handle = open(lockfile, 'w')
|
||||||
|
fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
yield
|
||||||
|
except BlockingIOError:
|
||||||
|
print('locked')
|
||||||
|
sys.exit(1)
|
|
@ -0,0 +1,16 @@
|
||||||
|
"""
|
||||||
|
WSGI config for logtracker project.
|
||||||
|
|
||||||
|
It exposes the WSGI callable as a module-level variable named ``application``.
|
||||||
|
|
||||||
|
For more information on this file, see
|
||||||
|
https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from django.core.wsgi import get_wsgi_application
|
||||||
|
|
||||||
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "logtracker.settings")
|
||||||
|
|
||||||
|
application = get_wsgi_application()
|
|
@ -0,0 +1,22 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "logtracker.settings")
|
||||||
|
try:
|
||||||
|
from django.core.management import execute_from_command_line
|
||||||
|
except ImportError:
|
||||||
|
# The above import may fail for some other reason. Ensure that the
|
||||||
|
# issue is really that Django is missing to avoid masking other
|
||||||
|
# exceptions on Python 2.
|
||||||
|
try:
|
||||||
|
import django
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError(
|
||||||
|
"Couldn't import Django. Are you sure it's installed and "
|
||||||
|
"available on your PYTHONPATH environment variable? Did you "
|
||||||
|
"forget to activate a virtual environment?"
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
execute_from_command_line(sys.argv)
|
|
@ -0,0 +1,134 @@
|
||||||
|
#! /usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from setuptools.command.install_lib import install_lib as _install_lib
|
||||||
|
from distutils.command.build import build as _build
|
||||||
|
from setuptools.command.sdist import sdist
|
||||||
|
from distutils.cmd import Command
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
|
||||||
|
class deb(sdist):
|
||||||
|
def run(self, *args):
|
||||||
|
os.system('git clean -fdx')
|
||||||
|
version = get_version()
|
||||||
|
sdist.run(self)
|
||||||
|
shutil.move("dist/logtracker-%s.tar.gz" % version, '../logtracker_%s.orig.tar.gz' % version)
|
||||||
|
os.system('git clean -fdx')
|
||||||
|
os.system('dpkg-source -b .')
|
||||||
|
os.system('sudo HOME=$HOME DIST=buster cowbuilder --build ../logtracker_%s-1.dsc' % version)
|
||||||
|
|
||||||
|
|
||||||
|
class eo_sdist(sdist):
|
||||||
|
def run(self):
|
||||||
|
if os.path.exists('VERSION'):
|
||||||
|
os.remove('VERSION')
|
||||||
|
version = get_version()
|
||||||
|
version_file = open('VERSION', 'w')
|
||||||
|
version_file.write(version)
|
||||||
|
version_file.close()
|
||||||
|
sdist.run(self)
|
||||||
|
if os.path.exists('VERSION'):
|
||||||
|
os.remove('VERSION')
|
||||||
|
|
||||||
|
|
||||||
|
def get_version():
|
||||||
|
'''Use the VERSION, if absent generates a version with git describe, if not
|
||||||
|
tag exists, take 0.0- and add the length of the commit log.
|
||||||
|
'''
|
||||||
|
if os.path.exists('VERSION'):
|
||||||
|
with open('VERSION', 'r') as v:
|
||||||
|
return v.read()
|
||||||
|
if os.path.exists('.git'):
|
||||||
|
p = subprocess.Popen(['git','describe','--dirty=.dirty','--match=v*'],
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
result = p.communicate()[0]
|
||||||
|
if p.returncode == 0:
|
||||||
|
result = result.decode('ascii').strip()[1:] # strip spaces/newlines and initial v
|
||||||
|
if '-' in result: # not a tagged version
|
||||||
|
real_number, commit_count, commit_hash = result.split('-', 2)
|
||||||
|
version = '%s.post%s+%s' % (real_number, commit_count, commit_hash)
|
||||||
|
else:
|
||||||
|
version = result
|
||||||
|
return version
|
||||||
|
else:
|
||||||
|
return '0.0.post%s' % len(
|
||||||
|
subprocess.check_output(
|
||||||
|
['git', 'rev-list', 'HEAD']).splitlines())
|
||||||
|
return '0.0'
|
||||||
|
|
||||||
|
|
||||||
|
class compile_translations(Command):
|
||||||
|
description = 'compile message catalogs to MO files via django compilemessages'
|
||||||
|
user_options = []
|
||||||
|
|
||||||
|
def initialize_options(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def finalize_options(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
try:
|
||||||
|
from django.core.management import call_command
|
||||||
|
for path, dirs, files in os.walk('fargo'):
|
||||||
|
if 'locale' not in dirs:
|
||||||
|
continue
|
||||||
|
curdir = os.getcwd()
|
||||||
|
os.chdir(os.path.realpath(path))
|
||||||
|
call_command('compilemessages')
|
||||||
|
os.chdir(curdir)
|
||||||
|
except ImportError:
|
||||||
|
sys.stderr.write('!!! Please install Django >= 1.4 to build translations\n')
|
||||||
|
|
||||||
|
|
||||||
|
class build(_build):
|
||||||
|
sub_commands = [('compile_translations', None)] + _build.sub_commands
|
||||||
|
|
||||||
|
|
||||||
|
class install_lib(_install_lib):
|
||||||
|
def run(self):
|
||||||
|
self.run_command('compile_translations')
|
||||||
|
_install_lib.run(self)
|
||||||
|
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name='logtracker',
|
||||||
|
version=get_version(),
|
||||||
|
description='Exim Logger',
|
||||||
|
long_description=open('README').read(),
|
||||||
|
author="Entr'ouvert",
|
||||||
|
author_email='info@entrouvert.com',
|
||||||
|
packages=find_packages(),
|
||||||
|
include_package_data=True,
|
||||||
|
scripts=('manage.py',),
|
||||||
|
url='https://dev.entrouvert.org/projects/logtracker/',
|
||||||
|
classifiers=[
|
||||||
|
'Development Status :: 2 - Pre-Alpha',
|
||||||
|
'Environment :: Web Environment',
|
||||||
|
'Framework :: Django',
|
||||||
|
'Intended Audience :: Developers',
|
||||||
|
'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
|
||||||
|
'Operating System :: OS Independent',
|
||||||
|
'Programming Language :: Python',
|
||||||
|
'Programming Language :: Python :: 2',
|
||||||
|
],
|
||||||
|
install_requires=[
|
||||||
|
'django>=1.11',
|
||||||
|
'psycopg2',
|
||||||
|
'django-mellon'
|
||||||
|
],
|
||||||
|
zip_safe=False,
|
||||||
|
cmdclass={
|
||||||
|
'build': build,
|
||||||
|
'compile_translations': compile_translations,
|
||||||
|
'install_lib': install_lib,
|
||||||
|
'sdist': eo_sdist,
|
||||||
|
'deb': deb,
|
||||||
|
},
|
||||||
|
)
|
|
@ -0,0 +1,3 @@
|
||||||
|
from django.test import TestCase
|
||||||
|
|
||||||
|
# Create your tests here.
|
|
@ -0,0 +1,48 @@
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pytest
|
||||||
|
import psycopg2
|
||||||
|
from django.db import connection
|
||||||
|
from django.contrib.auth.models import User
|
||||||
|
|
||||||
|
from logtracker.agent import journald, exim
|
||||||
|
|
||||||
|
|
||||||
|
basepath = os.path.dirname(__file__)
|
||||||
|
|
||||||
|
|
||||||
|
def run_sql(sql):
|
||||||
|
conn = psycopg2.connect(database='postgres')
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute(sql)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def journald_data(request, db, client):
|
||||||
|
with open('%s/testdata.journald' % basepath) as fh:
|
||||||
|
for line in fh.readlines():
|
||||||
|
parsed = journald.parse_line(line)
|
||||||
|
write(parsed)
|
||||||
|
|
||||||
|
|
||||||
|
def write(entry):
|
||||||
|
with connection.cursor() as c:
|
||||||
|
entry['data'] = json.dumps(entry['data'])
|
||||||
|
c.execute('''insert into collection_entry (host, service, priority, timestamp, data) values (%s, %s, %s, %s, %s)''',
|
||||||
|
(entry['host'], entry['service'], entry['priority'], entry['timestamp'], entry['data']))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def exim_data(request, db, client):
|
||||||
|
with open('%s/testdata.exim' % basepath) as fh:
|
||||||
|
for line in fh.readlines():
|
||||||
|
parsed = exim.parse_line(line)
|
||||||
|
if parsed:
|
||||||
|
write(parsed)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def auth(db, client):
|
||||||
|
User.objects.create_user(username='john', password='Doe')
|
||||||
|
client.login(username='john', password='Doe')
|
|
@ -0,0 +1,123 @@
|
||||||
|
"""
|
||||||
|
Django settings for logtracker project.
|
||||||
|
|
||||||
|
Generated by 'django-admin startproject' using Django 1.11.18.
|
||||||
|
|
||||||
|
For more information on this file, see
|
||||||
|
https://docs.djangoproject.com/en/1.11/topics/settings/
|
||||||
|
|
||||||
|
For the full list of settings and their values, see
|
||||||
|
https://docs.djangoproject.com/en/1.11/ref/settings/
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||||
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
|
||||||
|
# Quick-start development settings - unsuitable for production
|
||||||
|
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
|
||||||
|
|
||||||
|
# SECURITY WARNING: keep the secret key used in production secret!
|
||||||
|
SECRET_KEY = 'dxk)c9u(gq7@2tb)+*!=$09c=o0qnwbt^vrrmjcx-bsl&2!u7b'
|
||||||
|
|
||||||
|
# SECURITY WARNING: don't run with debug turned on in production!
|
||||||
|
DEBUG = True
|
||||||
|
|
||||||
|
ALLOWED_HOSTS = []
|
||||||
|
|
||||||
|
|
||||||
|
# Application definition
|
||||||
|
|
||||||
|
INSTALLED_APPS = [
|
||||||
|
'django.contrib.admin',
|
||||||
|
'django.contrib.auth',
|
||||||
|
'django.contrib.contenttypes',
|
||||||
|
'django.contrib.sessions',
|
||||||
|
'django.contrib.messages',
|
||||||
|
'django.contrib.staticfiles',
|
||||||
|
'logtracker.collection',
|
||||||
|
'logtracker.mail'
|
||||||
|
]
|
||||||
|
|
||||||
|
MIDDLEWARE = [
|
||||||
|
'django.middleware.security.SecurityMiddleware',
|
||||||
|
'django.contrib.sessions.middleware.SessionMiddleware',
|
||||||
|
'django.middleware.common.CommonMiddleware',
|
||||||
|
'django.middleware.csrf.CsrfViewMiddleware',
|
||||||
|
'django.contrib.auth.middleware.AuthenticationMiddleware',
|
||||||
|
'django.contrib.messages.middleware.MessageMiddleware',
|
||||||
|
'django.middleware.clickjacking.XFrameOptionsMiddleware',
|
||||||
|
]
|
||||||
|
|
||||||
|
ROOT_URLCONF = 'logtracker.urls'
|
||||||
|
|
||||||
|
TEMPLATES = [
|
||||||
|
{
|
||||||
|
'BACKEND': 'django.template.backends.django.DjangoTemplates',
|
||||||
|
'DIRS': [],
|
||||||
|
'APP_DIRS': True,
|
||||||
|
'OPTIONS': {
|
||||||
|
'context_processors': [
|
||||||
|
'django.template.context_processors.debug',
|
||||||
|
'django.template.context_processors.request',
|
||||||
|
'django.contrib.auth.context_processors.auth',
|
||||||
|
'django.contrib.messages.context_processors.messages',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
WSGI_APPLICATION = 'logtracker.wsgi.application'
|
||||||
|
|
||||||
|
|
||||||
|
# Database
|
||||||
|
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
|
||||||
|
|
||||||
|
DATABASES = {
|
||||||
|
'default': {
|
||||||
|
'ENGINE': 'django.db.backends.sqlite3',
|
||||||
|
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Password validation
|
||||||
|
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
|
||||||
|
|
||||||
|
AUTH_PASSWORD_VALIDATORS = [
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# Internationalization
|
||||||
|
# https://docs.djangoproject.com/en/1.11/topics/i18n/
|
||||||
|
|
||||||
|
LANGUAGE_CODE = 'en-us'
|
||||||
|
|
||||||
|
TIME_ZONE = 'UTC'
|
||||||
|
|
||||||
|
USE_I18N = True
|
||||||
|
|
||||||
|
USE_L10N = True
|
||||||
|
|
||||||
|
USE_TZ = True
|
||||||
|
|
||||||
|
|
||||||
|
# Static files (CSS, JavaScript, Images)
|
||||||
|
# https://docs.djangoproject.com/en/1.11/howto/static-files/
|
||||||
|
|
||||||
|
STATIC_URL = '/static/'
|
||||||
|
STATIC_ROOT = '/var/lib/logtracker/collectstatic'
|
|
@ -0,0 +1,5 @@
|
||||||
|
from logtracker.collection.models import Entry
|
||||||
|
|
||||||
|
|
||||||
|
def test_import_data(journald_data):
|
||||||
|
assert len(Entry.objects.all()) == 10
|
|
@ -0,0 +1,20 @@
|
||||||
|
import pytest
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
def test_auth(client, monkeypatch):
|
||||||
|
resp = client.get('/api/mail/')
|
||||||
|
assert resp.status_code == 302
|
||||||
|
|
||||||
|
|
||||||
|
def test_home_page(client, auth):
|
||||||
|
page = client.get('/')
|
||||||
|
assert page.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_entries_list(auth, client, journald_data):
|
||||||
|
page = client.get('/api/collection/')
|
||||||
|
assert page.status_code == 200
|
||||||
|
|
||||||
|
entries = json.loads(page.content.decode())
|
||||||
|
assert len(entries) == 10
|
|
@ -0,0 +1,17 @@
|
||||||
|
import pytest
|
||||||
|
import json
|
||||||
|
|
||||||
|
from django.contrib.auth.models import User
|
||||||
|
|
||||||
|
|
||||||
|
def test_mail_api(client, auth):
|
||||||
|
page = client.get('/api/mail/')
|
||||||
|
assert page.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_entries_list(auth, client, exim_data):
|
||||||
|
page = client.get('/api/collection/')
|
||||||
|
assert page.status_code == 200
|
||||||
|
|
||||||
|
entries = json.loads(page.content.decode())
|
||||||
|
assert len(entries) == 23
|
|
@ -0,0 +1,27 @@
|
||||||
|
2019-02-09 14:12:32 1gsSQq-0004Nr-HL <= ne-pas-repondre@chicago.fr H=my.server.net [5.135.221.10] P=esmtp S=8785 id=E1gsSQq-0001GI-EW@auquo.myserver.net
|
||||||
|
2019-02-09 14:12:32 1gsSQq-0004Nr-HL == john.guyot33@sfr.fr R=dnslookup T=remote_smtp defer (-53): retry time not reached for any host for 'sfr.fr'
|
||||||
|
2019-02-09 14:13:11 1gsSRS-0004Nu-Mk => ymelin@gmail.com R=dnslookup T=remote_smtp H=alt1.gmail-smtp-in.l.google.com [64.233.164.26] X=TLS1.2:ECDHE_RSA_CHACHA20_POLY1305:256 CV=yes DN="C=US,ST=California,L=Mountain View,O=Google LLC,CN=mx.google.com" C="250 2.0.0 OK 1549717991 u2-v6si1114219ljg.59 - gsmtp"
|
||||||
|
2019-02-09 14:13:11 1gsSRS-0004Nu-Mk Completed
|
||||||
|
2019-02-09 14:14:12 1gsSSS-0004Nz-6i <= ne-pas-repondre@departement666.fr H=my.server.net [5.135.221.10] P=esmtp S=7977 id=20190209131412.26403.28082@authentic.node1.prod.saas.myserver.net
|
||||||
|
2019-02-09 14:14:12 1gsSSS-0004Nz-6i ** dodu.jc06@hotmail.fr R=dnslookup T=remote_smtp H=eur.olc.protection.outlook.com [104.47.5.33] X=TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256 CV=yes DN="C=US,ST=Washington,L=Redmond,O=Microsoft Corporation,CN=mail.protection.outlook.com": SMTP error from remote mail server after RCPT TO:<dodu.jc06@hotmail.fr>: 550 5.5.0 Requested action not taken: mailbox unavailable.
|
||||||
|
2019-02-09 14:14:12 1gsSSS-0004O2-PV <= <> R=1gsSSS-0004Nz-6i U=Debian-exim P=local S=9550
|
||||||
|
2019-02-09 14:14:12 1gsSSS-0004Nz-6i Completed
|
||||||
|
2019-02-09 14:14:13 1gsSSS-0004O2-PV ** ne-pas-repondre@departement666.fr R=dnslookup T=remote_smtp H=mail-av5.departement666.fr [193.48.79.5]: SMTP error from remote mail server after RCPT TO:<ne-pas-repondre@departement666.fr>: 550 5.1.1 <ne-pas-repondre@departement666.fr>: Recipient address rejected: User unknown in relay recipient table
|
||||||
|
2019-02-09 14:14:13 1gsSSS-0004O2-PV Frozen (delivery error message)
|
||||||
|
2019-02-09 14:15:45 1gsSTx-0004OW-B7 <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=8433 id=20190209131545.4912.74435@authentic.myserver.net
|
||||||
|
2019-02-09 14:15:46 1gsSTx-0004OW-B7 => emmanuel@asfar.net R=dnslookup T=remote_smtp H=a.mx.asfar.net [69.168.84.3] C="250 ok 1549718146 qp 68658"
|
||||||
|
2019-02-09 14:15:46 1gsSTx-0004OW-B7 Completed
|
||||||
|
2019-02-09 14:18:25 1gsSWW-0004Ox-VJ <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=7971 id=20190209131824.8393.69975@authentic.myserver.net
|
||||||
|
2019-02-09 14:18:31 1gsSWd-0004P2-FY <= nepasrepondre@foret-sous-bois.fr H=my.server.net [5.135.221.10] P=esmtp S=8283 id=20190209131831.8393.49556@authentic.myserver.net
|
||||||
|
2019-02-09 14:18:32 1gsSWd-0004P2-FY SMTP error from remote mail server after end of data: 421-4.7.0 This message does not have authentication information or fails to pass\n421-4.7.0 authentication checks. To best protect our users from spam, the\n421-4.7.0 message has been blocked. Please visit\n421-4.7.0 https://support.google.com/mail/answer/81126#authentication for more\n421 4.7.0 information. q11si3575718wro.250 - gsmtp
|
||||||
|
2019-02-09 14:21:23 1gsSZP-0004PN-0x <= ne-pas-repondre+mymeaux@myserver.net H=my.server.net [5.135.221.10] P=esmtp S=6828 id=E1gsSZN-000x9T-ST@wcs.node1.prod.saas.myserver.net
|
||||||
|
2019-02-09 14:21:23 1gsSZP-0004PN-0x => a.desales@mars.planet <A.DeSales@mars.planet> R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery"
|
||||||
|
2019-02-09 14:21:23 1gsSZP-0004PN-0x -> a.obin@mars.planet <A.obin@mars.planet> R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery"
|
||||||
|
2019-02-09 14:21:23 1gsSZP-0004PN-0x -> athy.dick@mars.planet <Athy.Dick@mars.planet> R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery"
|
||||||
|
2019-02-09 14:21:23 1gsSZP-0004PN-0x -> aurelie.rihon@mars.planet <Aurelie.Rihon@mars.planet> R=dnslookup T=remote_smtp H=mx.relay.orange-business.com [194.2.0.84] C="250 2.0.0 x19DLNwZ201204 Message accepted for delivery"
|
||||||
|
2019-02-09 14:21:23 1gsSZP-0004PN-0x Completed
|
||||||
|
2019-02-09 17:25:30 H=(WIN-BR8CPBUF43R) [155.94.137.16] F=<spameri@tiscali.it> rejected RCPT <spameri@tiscali.it>: relay not permitted
|
||||||
|
2019-02-18 09:49:55 1gvecd-0008Vl-Jj H=netcourier.com [68.178.213.61]: Remote host closed connection in response to initial connection
|
||||||
|
2019-02-18 09:51:47 1gveaE-0008UG-HK H=alt2.gmail-smtp-in.l.google.com [2404:6800:4003:c02::1a] Connection timed out
|
||||||
|
2019-02-18 08:06:10 1gvbfc-0000qJ-BL H=mail.ualusa.com [168.235.95.126] Connection refused
|
||||||
|
2019-02-18 08:06:10 1gvcTW-0004qQ-RR Message is frozen
|
|
@ -0,0 +1,10 @@
|
||||||
|
{"_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","SYSLOG_PID":"5115","SYSLOG_TIMESTAMP":"Nov 8 14:45:01 ","_PID":"5115","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c406f;b=9b6948aea6564a80a6ecb7fc6684491a;m=bfcea5c9;t=596d5fd1133cf;x=1e677b790058a963","_UID":"0","_GID":"0","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","_TRANSPORT":"syslog","_SOURCE_REALTIME_TIMESTAMP":"1573220701505885","SYSLOG_IDENTIFIER":"CRON","__MONOTONIC_TIMESTAMP":"3217991113","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_HOSTNAME":"pad","__REALTIME_TIMESTAMP":"1573220701516751","SYSLOG_FACILITY":"9"}
|
||||||
|
{"__MONOTONIC_TIMESTAMP":"3217991316","_SYSTEMD_UNIT":"cron.service","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_UID":"0","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573220701516954","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4070;b=9b6948aea6564a80a6ecb7fc6684491a;m=bfcea694;t=596d5fd11349a;x=488c3f2177060430","_COMM":"cron","_AUDIT_LOGINUID":"0","_SYSTEMD_CGROUP":"/system.slice/cron.service","SYSLOG_FACILITY":"10","_SOURCE_REALTIME_TIMESTAMP":"1573220701511781","_GID":"0","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SYSTEMD_SLICE":"system.slice","SYSLOG_PID":"5114","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_TRANSPORT":"syslog","_CMDLINE":"/usr/sbin/CRON -f","PRIORITY":"6","_CAP_EFFECTIVE":"3fffffffff","MESSAGE":"pam_unix(cron:session): session closed for user root","_HOSTNAME":"pad","SYSLOG_IDENTIFIER":"CRON","_SELINUX_CONTEXT":"unconfined\n","_AUDIT_SESSION":"13","SYSLOG_TIMESTAMP":"Nov 8 14:45:01 ","_PID":"5114"}
|
||||||
|
{"MESSAGE":"Valid eCryptfs headers not found in file header region or xattr region, inode 7258970","_SOURCE_MONOTONIC_TIMESTAMP":"3779354493","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4071;b=9b6948aea6564a80a6ecb7fc6684491a;m=e1426d66;t=596d61e84fb6b;x=b17588d1ff1c6b03","PRIORITY":"7","SYSLOG_FACILITY":"0","_TRANSPORT":"kernel","SYSLOG_IDENTIFIER":"kernel","__MONOTONIC_TIMESTAMP":"3779226982","_HOSTNAME":"pad","__REALTIME_TIMESTAMP":"1573221262752619"}
|
||||||
|
{"__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4072;b=9b6948aea6564a80a6ecb7fc6684491a;m=e195117c;t=596d61ed79f82;x=a9d1e5633a803563","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","__MONOTONIC_TIMESTAMP":"3784642940","_TRANSPORT":"kernel","SYSLOG_IDENTIFIER":"kernel","PRIORITY":"7","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_HOSTNAME":"pad","SYSLOG_FACILITY":"0","MESSAGE":"Valid eCryptfs headers not found in file header region or xattr region, inode 10093410","__REALTIME_TIMESTAMP":"1573221268168578","_SOURCE_MONOTONIC_TIMESTAMP":"3784771755"}
|
||||||
|
{"PRIORITY":"6","SYSLOG_IDENTIFIER":"CRON","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","SYSLOG_FACILITY":"10","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_CAP_EFFECTIVE":"3fffffffff","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SELINUX_CONTEXT":"unconfined\n","_AUDIT_SESSION":"14","_SYSTEMD_SLICE":"system.slice","_AUDIT_LOGINUID":"0","_HOSTNAME":"pad","__MONOTONIC_TIMESTAMP":"3817997608","_UID":"0","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573221301523247","_PID":"5350","_TRANSPORT":"syslog","MESSAGE":"pam_unix(cron:session): session opened for user root by (uid=0)","SYSLOG_PID":"5350","_GID":"0","_SYSTEMD_UNIT":"cron.service","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_COMM":"cron","_SYSTEMD_CGROUP":"/system.slice/cron.service","_CMDLINE":"/usr/sbin/CRON -f","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4073;b=9b6948aea6564a80a6ecb7fc6684491a;m=e3920528;t=596d620d4932f;x=4d57d832d99b5cae","_SOURCE_REALTIME_TIMESTAMP":"1573221301523129"}
|
||||||
|
{"__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4074;b=9b6948aea6564a80a6ecb7fc6684491a;m=e3920819;t=596d620d4961f;x=f8784991133557ec","SYSLOG_PID":"5351","_SYSTEMD_UNIT":"cron.service","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","__REALTIME_TIMESTAMP":"1573221301523999","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_PID":"5351","_TRANSPORT":"syslog","_GID":"0","_SYSTEMD_CGROUP":"/system.slice/cron.service","_SELINUX_CONTEXT":"unconfined\n","_UID":"0","SYSLOG_FACILITY":"9","_HOSTNAME":"pad","_CMDLINE":"/usr/sbin/CRON -f","_CAP_EFFECTIVE":"3fffffffff","_EXE":"/usr/sbin/cron","_COMM":"cron","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_SOURCE_REALTIME_TIMESTAMP":"1573221301523811","SYSLOG_IDENTIFIER":"CRON","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_SYSTEMD_SLICE":"system.slice","_AUDIT_LOGINUID":"0","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","__MONOTONIC_TIMESTAMP":"3817998361","_AUDIT_SESSION":"14"}
|
||||||
|
{"__REALTIME_TIMESTAMP":"1573221301526069","SYSLOG_FACILITY":"10","_COMM":"cron","_SELINUX_CONTEXT":"unconfined\n","PRIORITY":"6","_SYSTEMD_SLICE":"system.slice","_SYSTEMD_CGROUP":"/system.slice/cron.service","_AUDIT_SESSION":"14","_HOSTNAME":"pad","_TRANSPORT":"syslog","_CMDLINE":"/usr/sbin/CRON -f","__MONOTONIC_TIMESTAMP":"3818000431","_SOURCE_REALTIME_TIMESTAMP":"1573221301526029","SYSLOG_PID":"5350","_CAP_EFFECTIVE":"3fffffffff","_SYSTEMD_UNIT":"cron.service","MESSAGE":"pam_unix(cron:session): session closed for user root","SYSLOG_IDENTIFIER":"CRON","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_GID":"0","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4075;b=9b6948aea6564a80a6ecb7fc6684491a;m=e392102f;t=596d620d49e35;x=d02be494a26c73d8","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_AUDIT_LOGINUID":"0","_UID":"0","_PID":"5350","SYSLOG_TIMESTAMP":"Nov 8 14:55:01 ","_EXE":"/usr/sbin/cron"}
|
||||||
|
{"_SYSTEMD_SLICE":"system.slice","_SOURCE_REALTIME_TIMESTAMP":"1573221901541654","_CAP_EFFECTIVE":"3fffffffff","SYSLOG_PID":"5620","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4076;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755941b;t=596d644982221;x=316c0c4c71f47f81","_GID":"0","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","_AUDIT_SESSION":"15","_UID":"0","_COMM":"cron","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_PID":"5620","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_CMDLINE":"/usr/sbin/CRON -f","_HOSTNAME":"pad","PRIORITY":"6","SYSLOG_IDENTIFIER":"CRON","_SYSTEMD_CGROUP":"/system.slice/cron.service","MESSAGE":"pam_unix(cron:session): session opened for user root by (uid=0)","_SELINUX_CONTEXT":"unconfined\n","_SYSTEMD_UNIT":"cron.service","_TRANSPORT":"syslog","SYSLOG_FACILITY":"10","__MONOTONIC_TIMESTAMP":"4418016283","_AUDIT_LOGINUID":"0","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","_EXE":"/usr/sbin/cron","__REALTIME_TIMESTAMP":"1573221901541921"}
|
||||||
|
{"_TRANSPORT":"syslog","SYSLOG_IDENTIFIER":"CRON","MESSAGE":"(root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)","_HOSTNAME":"pad","__MONOTONIC_TIMESTAMP":"4418027363","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","PRIORITY":"6","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","SYSLOG_FACILITY":"9","_PID":"5621","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","SYSLOG_PID":"5621","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4077;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755bf63;t=596d644984d69;x=cc55cc6eb1fab69e","_GID":"0","_UID":"0","_SOURCE_REALTIME_TIMESTAMP":"1573221901544064","__REALTIME_TIMESTAMP":"1573221901553001"}
|
||||||
|
{"SYSLOG_PID":"5620","_AUDIT_LOGINUID":"0","__MONOTONIC_TIMESTAMP":"4418027830","_SYSTEMD_UNIT":"cron.service","SYSLOG_FACILITY":"10","__CURSOR":"s=c911fd1c174a4fe8b1bec307d4a3886a;i=c4078;b=9b6948aea6564a80a6ecb7fc6684491a;m=10755c136;t=596d644984f3c;x=3b0e6ba1c3af0e15","_BOOT_ID":"9b6948aea6564a80a6ecb7fc6684491a","_SELINUX_CONTEXT":"unconfined\n","_EXE":"/usr/sbin/cron","_HOSTNAME":"pad","_GID":"0","PRIORITY":"6","_MACHINE_ID":"332e6893061344b3bf9e76f4f066f008","SYSLOG_IDENTIFIER":"CRON","MESSAGE":"pam_unix(cron:session): session closed for user root","__REALTIME_TIMESTAMP":"1573221901553468","_PID":"5620","_TRANSPORT":"syslog","SYSLOG_TIMESTAMP":"Nov 8 15:05:01 ","_COMM":"cron","_CAP_EFFECTIVE":"3fffffffff","_AUDIT_SESSION":"15","_SOURCE_REALTIME_TIMESTAMP":"1573221901550207","_SYSTEMD_INVOCATION_ID":"2ae92bce19144847b00ccd3ed137d5ec","_SYSTEMD_SLICE":"system.slice","_SYSTEMD_CGROUP":"/system.slice/cron.service","_UID":"0","_CMDLINE":"/usr/sbin/CRON -f"}
|
|
@ -0,0 +1,28 @@
|
||||||
|
# Tox (http://tox.testrun.org/) is a tool for running tests
|
||||||
|
# in multiple virtualenvs. This configuration file will run the
|
||||||
|
# test suite on all supported python versions. To use it, "pip install tox"
|
||||||
|
# and then run "tox" from this directory.
|
||||||
|
|
||||||
|
[tox]
|
||||||
|
toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/logtracker/{env:BRANCH_NAME:}
|
||||||
|
envlist = dj111
|
||||||
|
|
||||||
|
[testenv]
|
||||||
|
usedevelop = true
|
||||||
|
basepython = python3
|
||||||
|
setenv =
|
||||||
|
DJANGO_SETTINGS_MODULE=tests.settings
|
||||||
|
coverage: COVERAGE=--junit-xml=test_results.xml --cov=logtracker --cov-report xml
|
||||||
|
deps =
|
||||||
|
dj111: django>=1.11,<1.12
|
||||||
|
coverage
|
||||||
|
psycopg2
|
||||||
|
pytest
|
||||||
|
pytest-cov
|
||||||
|
pytest-django
|
||||||
|
tz
|
||||||
|
commands =
|
||||||
|
dj111: py.test-3 {posargs: --junitxml=test_{envname}_results.xml --cov-report xml --cov-report html --cov=logtracker -s -x tests/ --disable-warnings}
|
||||||
|
[pytest]
|
||||||
|
filterwarnings =
|
||||||
|
once:.*
|
Reference in New Issue