78 lines
3.3 KiB
Python
Executable File
78 lines
3.3 KiB
Python
Executable File
#!/usr/bin/python3
|
|
# Entrouvert 2019
|
|
# Exim log parser
|
|
# See Summary of Fields in Log Lines in https://www.exim.org/exim-html-current/doc/html/spec_html/ch-log_files.html
|
|
import copy
|
|
import re
|
|
import datetime
|
|
import json
|
|
import pytz
|
|
|
|
from django.utils import timezone
|
|
|
|
from logtracker.journal.models import Entry
|
|
from logtracker.mail.models import Mail, Property, Sender, Recipient
|
|
|
|
|
|
paris = pytz.timezone('Europe/Paris')
|
|
patterns = {'ignore': re.compile('([\d-]+) ([\d:]+) .*(Start queue run|End queue run|daemon started|relay not permitted|Spool file is locked|Connection refused|Connection timed out|no immediate delivery|error ignored|Greylisting in action|Remote host closed connection|No route to host|SMTP error|SMTP protocol error|SMTP protocol synchronization error|SMTP command timeout|no host name found|unexpected disconnection|TLS error|log string overflowed|cancelled by timeout).*'),
|
|
'match': re.compile('(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) ([\w\d\-]+) (<=|=>|->|==|\*\*|Completed|SMTP error|Message is frozen|Frozen|Unfrozen)\s*(.*)$'),
|
|
}
|
|
|
|
|
|
|
|
def parse_date(string):
|
|
stamp = datetime.datetime.strptime(string, '%Y-%m-%d %H:%M:%S')
|
|
return timezone.make_aware(stamp, paris)
|
|
|
|
|
|
def parse_exim_line(line):
|
|
match = re.match(patterns['match'], line)
|
|
if match:
|
|
stamp, identifier, action, data = match.groups()
|
|
if 'domain matches queue_smtp_domains, or -odqs set' in data:
|
|
return
|
|
stamp = parse_date(stamp)
|
|
return {'stamp': stamp, 'identifier': identifier, 'action': action, 'data': data[:511]}
|
|
else:
|
|
match = re.match(patterns['ignore'], line)
|
|
if not match:
|
|
print('Failed to parse line: %s' % line)
|
|
|
|
|
|
def extract_exim_emails():
|
|
metadata, _ = Property.objects.get_or_create(name='metadata')
|
|
if 'extracted_index' not in metadata.data:
|
|
metadata.data = json.loads('{"extracted_index": 0}')
|
|
metadata.save()
|
|
e = None
|
|
try:
|
|
for e in Entry.objects.filter(pk__gt=metadata.data['extracted_index']):
|
|
data = json.loads(e.data)
|
|
if data.get('_COMM') in ['exim', 'exim4']:
|
|
entry = parse_exim_line(data.get('MESSAGE'))
|
|
if not entry:
|
|
continue
|
|
mail, created = Mail.objects.get_or_create(identifier = entry['identifier'])
|
|
if created:
|
|
mail.stamp = entry['stamp']
|
|
if entry['action'] == '<=':
|
|
address = entry['data'].split(' ')[0]
|
|
sender, _ = Sender.objects.get_or_create(email=address)
|
|
mail.sender = sender
|
|
elif entry['action'] in ('=>', '->'):
|
|
address, _ = Recipient.objects.get_or_create(email=entry['data'].split(' ')[0])
|
|
mail.recipients.add(address)
|
|
elif entry['action'] == 'Completed':
|
|
mail.has_completed = True
|
|
elif entry['action'] == '==' or entry['action'] == '**':
|
|
mail.has_error = True
|
|
entries = copy.deepcopy(mail.entries)
|
|
entries.append(e.pk)
|
|
mail.entries = entries
|
|
mail.save()
|
|
finally:
|
|
if e:
|
|
metadata.data['extracted_index'] = e.pk
|
|
metadata.save()
|