From cecd8c30d3c6ceaadc89c7be1ed2ae304f89d32e Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Mon, 9 Sep 2013 17:45:35 +0200 Subject: [PATCH] pfwb: overload docbow command sendmail() This command handle specificities about the handling of incoming emails at the PFWB. It uses six new special application settings. The loading order of the pfwb application was changed as you can only overload a command from a previously loaded application. ref #3542 --- docbow_project/pfwb/README.txt | 19 +- docbow_project/pfwb/app_settings.py | 10 + docbow_project/pfwb/management/__init__.py | 0 .../pfwb/management/commands/__init__.py | 0 .../pfwb/management/commands/sendmail.py | 230 ++++++++++++++++++ docbow_project/settings.py | 10 +- 6 files changed, 266 insertions(+), 3 deletions(-) create mode 100644 docbow_project/pfwb/management/__init__.py create mode 100644 docbow_project/pfwb/management/commands/__init__.py create mode 100644 docbow_project/pfwb/management/commands/sendmail.py diff --git a/docbow_project/pfwb/README.txt b/docbow_project/pfwb/README.txt index 6613cb3..bc786d8 100644 --- a/docbow_project/pfwb/README.txt +++ b/docbow_project/pfwb/README.txt @@ -9,6 +9,9 @@ attached file and json file describing the document. Settings ======== +GED export +---------- + You must add the following lines to your local_settings.py file:: from docbow_project.settings.dev import INSTALLED_APPS @@ -17,7 +20,21 @@ You must add the following lines to your local_settings.py file:: Then you must define in the same local_settings.py file the directory where new attached files will be written, for example:: - PFWB_GED_DIRECTORY = '/var/lib/plone/docbow_import_directory/' + DOCBOW_PFWB_GED_DIRECTORY = '/var/lib/plone/docbow_import_directory/' + +SMTP interface +-------------- + +PFWB_SENDMAIL_DEFAULT_TYPE_ID: default FileType id for mails with unknown filetype +PFWB_SENDMAIL_DEFAULT_TYPE_NAME: default FileType name to create if no default FileType id exists +PFWB_SENDMAIL_TABELLIO_EXPEDITION_EMAIL: email to match for accepting mails +coming from Tabellio expedition +PFWB_SENDMAIL_TABELLIO_EXPEDITION_USER_ID: id of the user to assign to document +received from tabellio expedition +PFWB_SENDMAIL_ATTACHED_FILE_EMAIL: email to match for accepting mails with +files attached +PFWB_SENDMAIL_ATTACHED_FILE_USER_ID: user id of the user to assign to document +received with files attached File naming =========== diff --git a/docbow_project/pfwb/app_settings.py b/docbow_project/pfwb/app_settings.py index 2abe9bd..93728b4 100644 --- a/docbow_project/pfwb/app_settings.py +++ b/docbow_project/pfwb/app_settings.py @@ -1,3 +1,13 @@ from django.conf import settings PFWB_GED_DIRECTORY = getattr(settings, 'DOCBOW_PFWB_GED_DIRECTORY', None) +PFWB_SENDMAIL_DEFAULT_TYPE_ID = getattr(settings, 'DOCBOW_PFWB_SENDMAIL_DEFAULT_TYPE_ID', None) +PFWB_SENDMAIL_DEFAULT_TYPE_NAME = getattr(settings, 'DOCBOW_PFWB_SENDMAIL_DEFAULT_TYPE_NAME', 'Divers') +PFWB_SENDMAIL_TABELLIO_EXPEDITION_EMAIL = getattr(settings, + 'DOCBOW_PFWB_SENDMAIL_TABELLIO_EXPEDITION_EMAIL', 'commande.documents@pfwb.be') +PFWB_SENDMAIL_TABELLIO_EXPEDITION_USER_ID = getattr(settings, + 'DOCBOW_PFWB_SENDMAIL_TABELLIO_EXPEDITION_USER_ID', None) +PFWB_SENDMAIL_ATTACHED_FILE_EMAIL = getattr(settings, + 'DOCBOW_PFWB_SENDMAIL_ATTACHED_FILE_EMAIL', None) +PFWB_SENDMAIL_ATTACHED_FILE_USER_ID = getattr(settings, + 'DOCBOW_PFWB_SENDMAIL_ATTACHED_FILE_USER_ID', None) diff --git a/docbow_project/pfwb/management/__init__.py b/docbow_project/pfwb/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/docbow_project/pfwb/management/commands/__init__.py b/docbow_project/pfwb/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/docbow_project/pfwb/management/commands/sendmail.py b/docbow_project/pfwb/management/commands/sendmail.py new file mode 100644 index 0000000..2f78368 --- /dev/null +++ b/docbow_project/pfwb/management/commands/sendmail.py @@ -0,0 +1,230 @@ +from optparse import make_option +import sys +import email +import email.errors +import email.utils +import email.header +import logging +import re +import urllib2 + +from django.core.management.base import BaseCommand, CommandError +import django.contrib.auth.models as auth_models +from django.core.files.base import ContentFile +from django.db import transaction +from django.core.exceptions import MultipleObjectsReturned + +from docbow_project.docbow import models, views, timestamp +from docbow_project.docbow.email_utils import u2u_decode +from docbow_project.docbow.app_settings import BASE_URL +from django_journal import record + +from ... import app_settings, models as pfwb_models + +logger = logging.getLogger('docbow.mail_interface') + +# modes +EXPEDITION = 'expedition' +ATTACHED_FILE = 'attached_file' + +class Command(BaseCommand): + args = '' + help = '''Convert a mail to a document send. + +In case of failure the following return value is returned: + - 1 failure to parse the message on stdin + - 2 the mail is missing a subject + - 3 the subject could not be decoded + - 4 obligatory attributes or contents were missing + - 5 notifications mails could not be sent + - 6 missing Message-ID + - 7 sender not authorized + - 8 missing sender +''' + + option_list = BaseCommand.option_list + ( + make_option("--ip", default=''), + make_option("--sender"), + make_option("--file")) + + def handle(self, *args, **options): + if not options.get('sender'): + self.error('7.7.1 No sender', exit_code=8) + try: + if options.get('file'): + mail = email.message_from_file(file(options['file'])) + else: + mail = email.message_from_file(sys.stdin) + except email.errors.MessageParseError, e: + self.error('7.7.1 Error parsing message', exite_code=1) + try: + self.handle_mail(mail, args, **options) + except Exception, e: + logger.exception('Unknown exception') + self.error('7.7.1 Internal error when handling the mail', exit_code=5) + + def error(self, msg, exit_code=None, **kwargs): + print >>sys.stderr, msg.format(**kwargs) + if hasattr(self, 'message_id'): + record('error-smtp-interface', 'message {message_id} refused: ' + msg, + message_id=self.message_id, **kwargs) + else: + record('error-smtp-interface', 'message refused: ' + msg, **kwargs) + if exit_code: + sys.exit(exit_code) + + def decode_filename(self, filename): + '''See if the filename contains encoded-word work around bugs in FileMakerPro''' + m = re.match(r'=\?(.*)\?(.*)\?(.*)\?=', filename) + if m: + result = [] + for content, encoding in email.header.decode_header(filename): + result.append(unicode(content, encoding or 'ascii')) + return ''.join(result) + else: + return filename + + @transaction.commit_on_success + def handle_mail(self, mail, mail_recipients, **options): + content_errors = [] + attachments = [] + recipients = [] + description = u'' + from_email = email.utils.parseaddr(options['sender'])[1] + if not from_email: + self.error('7.7.1 No sender', exit_code=8) + if from_email == app_settings.PFWB_SENDMAIL_TABELLIO_EXPEDITION_EMAIL: + mode = EXPEDITION + elif from_email == app_settings.PFWB_SENDMAIL_ATTACHED_FILE_EMAIL: + mode = ATTACHED_FILE + else: + self.error('Email unknown, not authorized to post', exit_code=7) + return + tos = mail.get_all('to', []) + ccs = mail.get_all('cc', []) + resent_tos = mail.get_all('resent-to', []) + resent_ccs = mail.get_all('resent-cc', []) + all_recipients = mail_recipients or [b for a, b in email.utils.getaddresses(tos + ccs + resent_tos + + resent_ccs)] + self.message_id = mail.get('Message-ID', None) + if not self.message_id: + self.error('7.7.1 Mail is missing a Message-ID', exit_code=6) + + # determine the filetype + filetype = None + if mode == ATTACHED_FILE: + subject = mail.get('Subject', None) + if not subject: + self.error('7.7.1 Mail is missing a subject', exit_code=2) + try: + subject = u2u_decode(subject) + except Exception, e: + self.error('7.7.1 The subject cannot be decoded', exit_code=3) + try: + filetype = models.FileType.objects.get(name=subject) + except models.FileType.DoesNotExist: + record('warning', 'unknown filetype ' + '{filetype}, using default filetype', + filetype=subject) + else: + tabellio_doc_type = mail.get('x-tabellio-doc-type') + if tabellio_doc_type: + try: + filetype = models.FileType.objects.get( + tabelliodoctype__tabellio_doc_type=tabellio_doc_type) + except models.FileType.DoesNotExist: + record('warning', 'unknown x-tabellio-doc-type ' + '{tabellio_doc_type}, using default filetype', + tabellio_doc_type=tabellio_doc_type) + except models.FileType.MultipleObjectsReturned: + record('warning', 'unknown x-tabellio-doc-type ' + '{tabellio_doc_type}, using default filetype', + tabellio_doc_type=tabellio_doc_type) + if filetype is None: + try: + filetype = models.FileType.objects.get( + id=app_settings.PFWB_SENDMAIL_DEFAULT_TYPE_ID) + except models.FileType.DoesNotExist: + filetype, created = models.FileType.objects.get_or_create( + name=app_settings.PFWB_SENDMAIL_DEFAULT_TYPE_NAME) + + if mode == ATTACHED_FILE: + for part in mail.walk(): + filename = part.get_filename(None) + if part.get_content_type() == 'text/plain' and \ + ('Content-Disposition' not in part or 'inline' in part['Content-Disposition']): + charset = part.get_content_charset('us-ascii') + description = unicode(part.get_payload(decode=True), charset) + + if filename: + attachments.append((self.decode_filename(filename), + part.get_payload(decode=True))) + else: + url = mail.get('x-tabellio-doc-url') + stream = urllib2.urlopen(url) + content = stream.read() + subject = mail.get('Subject', None) + if not subject: + self.error('7.7.1 Mail is missing a subject', exit_code=2) + try: + subject = u2u_decode(subject) + except Exception, e: + self.error('7.7.1 The subject cannot be decoded', exit_code=3) + try: + name = subject.split(':', 1)[1].strip() + except IndexError: + self.error('7.7.1 Filename cannot be extracted from the subject', exit_code=3) + attachments.append((name, content)) + + for email_address in all_recipients: + try: + user = auth_models.User.objects.get(email=email_address, delegations_by__isnull=True) + recipients.append(user) + except auth_models.User.DoesNotExist: + msg = 'Recipient %r is not an user of the platform' \ + % email_address + content_errors.append(msg) + except MultipleObjectsReturned: + msg = 'Recipient %r has more than 1 user in the platform' \ + % email_address + content_errors.append(msg) + if not len(attachments): + content_errors.append('You must send at least one attached file') + if not len(all_recipients): + content_errors.append('You must have at least one recipient in your message.') + try: + if mode == ATTACHED_FILE: + user_id = app_settings.PFWB_SENDMAIL_ATTACHED_FILE_USER_ID + else: + user_id = app_settings.PFWB_SENDMAIL_TABELLIO_EXPEDITION_USER_ID + sender = auth_models.User.objects.get(id=user_id) + except auth_models.User.DoesNotExist: + content_errors.append('No user match the sender user_id %s in mode ' + '%s' % (user_id, mode)) + if content_errors: + msg = [ '7.7.1 The email sent contains many errors:' ] + for error in content_errors: + msg.append(' - %s' % error) + self.error('\n'.join(msg), exit_code=4) + else: + if mode == ATTACHED_FILE: + record('smtp-received-document', 'mode: {mode} message-id: {message_id} subject: {subject}', + mode=mode, message_id=self.message_id, subject=subject) + else: + record('smtp-received-document', 'mode: {mode} message-id: ' + '{message_id} x-tabellio-doc-url: {x_tabellio_doc_url} ' + 'x-tabellio-doc-type: {x_tabellio_doc_type}', + mode=mode, message_id=self.message_id, + subject=subject, x_tabellio_doc_url=url, + x_tabellio_doc_type=tabellio_doc_type) + document = models.Document(sender=sender, + comment=description, filetype=filetype) + document.save() + document.to_user = recipients + for filename, payload in attachments: + content = ContentFile(payload) + attached_file = models.AttachedFile(document=document, + name=filename) + attached_file.content.save(filename, content, save=False) + attached_file.save() + document.post() diff --git a/docbow_project/settings.py b/docbow_project/settings.py index 0685c46..1096295 100644 --- a/docbow_project/settings.py +++ b/docbow_project/settings.py @@ -62,11 +62,17 @@ __ENVIRONMENT_DEFAULTS = dict( DOCBOW_SMS_CARRIER_CLASS='docbow_project.docbow.sms_carrier_ovh.OVHSMSCarrier', DOCBOW_ORGANIZATION_SHORT='PFWB', DOCBOW_ORGANIZATION=u'Parlement de la Fédération Wallonie-Bruxelles', - DOCBOW_PFWB_GED_DIRECTORY='/var/lib/%s/ged/', DOCBOW_NOTIFIERS=( 'docbow_project.docbow.notification.MailNotifier', 'docbow_project.docbow.notification.SMSNotifier', ), + DOCBOW_PFWB_GED_DIRECTORY='/var/lib/%s/ged/', + DOCBOW_PFWB_SENDMAIL_DEFAULT_TYPE_ID=None, + DOCBOW_PFWB_SENDMAIL_DEFAULT_TYPE_NAME=u'Divers', + DOCBOW_PFWB_SENDMAIL_TABELLIO_EXPEDITION_EMAIL='commande.documents@pfwb.be', + DOCBOW_PFWB_SENDMAIL_TABELLIO_EXPEDITION_USER_ID=None, + DOCBOW_PFWB_SENDMAIL_ATTACHED_FILE_EMAIL='dontknow@pfwb.be', + DOCBOW_PFWB_SENDMAIL_ATTACHED_FILE_USER_ID=None, TEMPLATE_CONTEXT_PROCESSORS=django.conf.global_settings.TEMPLATE_CONTEXT_PROCESSORS+('django.core.context_processors.request',), DATE_INPUT_FORMATS=('%d/%m/%Y', '%Y-%m-%d'), CRISPY_TEMPLATE_PACK='uni_form', @@ -126,8 +132,8 @@ MIDDLEWARE_CLASSES = ( ) INSTALLED_APPS = ( - 'docbow_project.pfwb', 'docbow_project.docbow', + 'docbow_project.pfwb', 'django_tables2', 'grappelli', 'django_journal',