maarch: merge minint modifications (#41283)

This commit is contained in:
Thomas NOËL 2020-05-02 22:41:28 +02:00
parent 2183f49d1d
commit 283dc82970
3 changed files with 527 additions and 9 deletions

View File

@ -10,6 +10,10 @@
>{{ site_base_uri }}{% url 'maarch-ping' slug=object.slug %}</a>[?debug]</li>
<li>{% trans 'Store a resource:' %} POST <a href="{% url 'maarch-resource' slug=object.slug %}"
>{{ site_base_uri }}{% url 'maarch-resource' slug=object.slug %}</a> (payload: JSON w.c.s. formdata)</li>
<li>Ajout d'un fichier à un dossier existant POST <a href="{% url 'maarch-attachment' slug=object.slug %}"
>{{ site_base_uri }}{% url 'maarch-attachment' slug=object.slug %}</a> (payload: JSON w.c.s. formdata)</li>
<li>Debug de maarch-resource POST <a href="{% url 'maarch-debug' slug=object.slug %}"
>{{ site_base_uri }}{% url 'maarch-debug' slug=object.slug %}</a> (payload: JSON w.c.s. formdata)</li>
</ul>
{% endblock %}

View File

@ -25,4 +25,8 @@ urlpatterns = [
name='maarch-ping'),
url(r'^(?P<slug>[\w,-]+)/resource/$', ResourceView.as_view(),
name='maarch-resource'),
url(r'^(?P<slug>[\w,-]+)/attachment/$', AttachmentView.as_view(),
name='maarch-attachment'),
url(r'^(?P<slug>[\w,-]+)/debug/$', DebugView.as_view(),
name='maarch-debug'),
]

View File

@ -17,6 +17,8 @@
import requests
from datetime import datetime
import logging
import re
import string
from django.views.generic import DetailView as GenericDetailView
from django.utils.decorators import method_decorator
@ -26,6 +28,7 @@ from django.views.decorators.csrf import csrf_exempt
from passerelle.compat import json_loads
import passerelle.utils as utils
from passerelle.soap import sudsobject_to_dict, client_to_jsondict
from suds import TypeNotFound
from .soap import get_client
from .models import MinIntMaarch
@ -42,6 +45,7 @@ class MaarchDetailView(GenericDetailView):
class DetailView(GenericDetailView):
model = MinIntMaarch
translations = False
def get_client(self):
return get_client(self.get_object())
@ -54,6 +58,15 @@ class DetailView(GenericDetailView):
data = self.get_data(request, *args, **kwargs)
return utils.response_for_json(request, data)
def prepare_regexp_filename_replacement(self, extras):
pattern = extras.get('filename_search_pattern_in_label')
result = extras.get('filename_replace_pattern')
if pattern and result:
return re.compile(pattern), result
return False, False
def translate(self, key, default):
return self.translations[key] if self.translations and key in self.translations else default
class PingView(DetailView):
def get_data(self, request, *args, **kwargs):
@ -91,14 +104,24 @@ class ResourceView(DetailView):
# storeAttachmentResource attachments: list, build from formdata file fields
attachments = []
# regexp to rename attachments
regexp_filename_search_pattern_in_label, filename_result_pattern = self.prepare_regexp_filename_replacement(
extras)
#
# build document (encodedFile and fileFormat)
#
# get translations
translations_txt = extras.get('translations')
if translations_txt:
self.translations = json.loads(translations_txt)
# logger.debug('Translation JSON: %r', translations)
document = u'<html>'
document += u'<head><meta charset="utf-8"></head>'
document += '<body>'
document += u'<h1>%s</h1>' % schema['name']
document += u'<h1>%s</h1>' % self.translate('form_title', schema['name'])
page = ''
empty_page = True
for field in schema['fields']:
@ -106,10 +129,12 @@ class ResourceView(DetailView):
# add last page, if it contains values
if page and not empty_page:
document += page
page = u'<hr /><h2>%s</h2>' % field['label']
page = u'<hr /><h2>%s</h2>' % self.translate(field['label'], field['label'])
empty_page = True
elif field['type'] == 'title':
page += u'<h3>%s</h3>' % field['label']
# elif field['type'] == 'subtitle':
# page += u'<h3>%s</h3>' % self.translate(field['label'], field['label'])
elif 'varname' in field:
varname = field['varname']
value = formdata['fields'].get(varname)
@ -118,10 +143,23 @@ class ResourceView(DetailView):
if field['type'] == 'file':
# field is a file: add it to attachments list
value['fileFormat'] = value['content_type'].split('/')[1] # FIXME (how ?)
if regexp_filename_search_pattern_in_label:
m = regexp_filename_search_pattern_in_label.match(field['label'])
if m:
value['filename_original'] = value['filename']
value['filename'] = string.replace(filename_result_pattern, 'searchedPatternInLabel', m.group(1))
if 'filename' in filename_result_pattern:
value['filename'] = string.replace(value['filename'], 'filename', value['filename_original'])
attachments.append(value)
value = '%s' % value['filename']
elif field['type'] == 'date':
value = datetime.strptime(value, '%Y-%m-%d')
value = value.strftime('%d-%m-%Y')
elif field['type'] == 'item' and self.translations:
value_raw = formdata['fields'].get(varname + '_raw')
value = self.translate(varname + '_' + value_raw, value)
page += u'<dl>'
page += u'<dt>%s</dt>' % field['label']
page += u'<dt>%s</dt>' % self.translate(varname, field['label'])
page += u'<dd>%s</dd>' % value
page += u'</dl>'
empty_page = False
@ -142,14 +180,47 @@ class ResourceView(DetailView):
metadata = {}
# storeExtResource metadata
ext_metadata = {}
# createContact data
contact = {}
contact_table1_pp = {'title', 'lastname', 'firstname', 'contact_type', 'is_corporate_person'}
# extract metadata and ext_metadata from formdata['extra']
for name, value in extras.items():
if name.startswith('maarch_contact_'):
contact[name[15:]] = value
if name.startswith('maarch_metadata_'):
metadata[name[16:]] = value
if name.startswith('maarch_ext_metadata_'):
ext_metadata[name[20:]] = value
def prepare_soap_contact(contact, contact_table1_pp):
datas = []
for name, value in contact.items():
data = client.factory.create('arrayOfDataContactContent')
data.column = name
if name.endswith('_date'):
data.type = 'date'
value = datetime.strptime(value[:19], '%Y-%m-%dT%H:%M:%S')
value = value.strftime('%d-%m-%Y %H:%M:%S')
elif isinstance(value, basestring):
data.type = 'string'
elif isinstance(value, int):
data.type = 'integer'
elif isinstance(value, float):
data.type = 'float'
elif value is None:
data.type = 'string'
value = ''
data.value = value
if name in contact_table1_pp:
data.table = 'contacts_v2'
else:
data.table = 'contact_addresses'
datas.append(data)
soap_contact = client.factory.create('arrayOfDataContact')
soap_contact.datas = datas
return soap_contact
# prepare metadata for SOAP call
def prepare_soap_metadata(metadata):
datas = []
@ -175,12 +246,24 @@ class ResourceView(DetailView):
soap_metadata.datas = datas
return soap_metadata
logger.debug('Extraction DATA contact %r: ', contact)
logger.debug('Extraction DATA metadata %r: ', metadata)
# if INES ESB (Tibco) between passerelle and Maarch, add a "maarch_id"
# parameter. Get value from formdata or workflow options.
maarch_id = extras.get('maarch_id')
if maarch_id:
contact = False
else:
contact = prepare_soap_contact(contact, contact_table1_pp)
metadata = prepare_soap_metadata(metadata)
ext_metadata = prepare_soap_metadata(ext_metadata)
if debug:
debug_output['contact'] = '%r' % contact
debug_output['metadata'] = '%r' % metadata
debug_output['ext_metadata'] = '%r' % ext_metadata
#
# get other Maarch variables (letterbox configuration by default)
@ -197,9 +280,7 @@ class ResourceView(DetailView):
debug_output['status'] = status
debug_output['ext_table'] = ext_table
# if INES ESB (Tibco) between passerelle and Maarch, add a "maarch_id"
# parameter. Get value from formdata or workflow options.
maarch_id = extras.get('maarch_id')
if debug:
debug_output['maarch_id'] = maarch_id
@ -207,8 +288,32 @@ class ResourceView(DetailView):
#
# call Maarch web services
#
logger.debug('createContact: start')
if maarch_id:
logger.debug('createContact : SKIPPED done by INES for maarch_id: %r', maarch_id)
data_contact = False
else:
try:
create_contact_result = client.service.CreateContact(contact)
data_contact = sudsobject_to_dict(create_contact_result)
logger.debug('createContact result: %r', data_contact)
except TypeNotFound:
pass
logger.debug('createContact: end')
logger.debug('storeResource+Ext+Attachment: start')
# Prepare ext_metadata
if data_contact:
ext_metadata['exp_contact_id'] = data_contact['contactId']
ext_metadata['address_id'] = data_contact['addressId']
ext_metadata = prepare_soap_metadata(ext_metadata)
logger.debug('Extraction DATA ext_metadata %r: ', ext_metadata)
if debug:
debug_output['ext_metadata'] = '%r' % ext_metadata
# store the resource (storeResource)
logger.debug('storeResource: encodedFile(size):%r fileFormat:%r '
'collId:%r table:%r status:%r', len(encodedFile),
@ -234,7 +339,7 @@ class ResourceView(DetailView):
logger.debug('storeExtResource: resId:%r ext_table:%r', resId,
ext_table)
logger.debug('storeExtResource: ext_metadata: %r', ext_metadata)
# logger.debug('storeExtResource: ext_metadata: %r', ext_metadata)
# store external metadata (storeExtResource)
if maarch_id:
logger.debug('storeExtResource: INES maarch_id: %r', maarch_id)
@ -267,3 +372,408 @@ class ResourceView(DetailView):
logger.debug('storeResource+Ext+Attachment: resId:%r -- end', resId)
return utils.response_for_json(request, data)
class AttachmentView(DetailView):
@method_decorator(csrf_exempt)
def dispatch(self, *args, **kwargs):
return super(AttachmentView, self).dispatch(*args, **kwargs)
@utils.protected_api('can_access')
def post(self, request, *args, **kwargs):
client = self.get_client()
formdata = json.loads(request.body)
extras = formdata.get('extra', {})
debug = 'debug' in request.GET
if debug:
debug_output = {}
# get formdef schema from wcs API
url = formdata['url']
p = urlparse.urlsplit(url)
scheme, netloc, path, query, fragment = \
p.scheme, p.netloc, p.path, p.query, p.fragment
schema_path = path.rsplit('/', 2)[0] + '/schema'
schema_url = urlparse.urlunsplit((scheme, netloc, schema_path, query, fragment))
schema = requests.get(schema_url).json()
regexp_filename_search_pattern_in_label, filename_result_pattern = self.prepare_regexp_filename_replacement(extras)
# storeAttachmentResource attachments: list, build from formdata file fields
attachments = []
for field in schema['fields']:
if 'varname' in field:
varname = field['varname']
value = formdata['fields'].get(varname)
if not value:
continue
if field['type'] == 'file':
# field is a file: add it to attachments list
value['fileFormat'] = value['content_type'].split('/')[1] # FIXME (how ?)
if regexp_filename_search_pattern_in_label:
m = regexp_filename_search_pattern_in_label.match(field['label'])
if m:
value['filename_original'] = value['filename']
value['filename'] = string.replace(filename_result_pattern, 'searchedPatternInLabel', m.group(1))
if 'filename' in filename_result_pattern:
value['filename'] = string.replace(value['filename'], 'filename', value['filename_original'])
attachments.append(value)
#
# get other Maarch variables (letterbox configuration by default)
#
resId = extras.get('maarch_resId') or 0
collId = extras.get('maarch_collId') or 'letterbox_coll'
data = {}
if debug:
debug_output['resId'] = resId
debug_output['collId'] = collId
# if INES ESB (Tibco) between passerelle and Maarch, add a "maarch_id"
# parameter. Get value from formdata or workflow options.
maarch_id = extras.get('maarch_id')
notification_email = extras.get('notification_email') if maarch_id else None
if debug:
debug_output['maarch_id'] = maarch_id
logger.debug('storeAttachment: start')
debug_output['res'] = []
# store attachments
nbAttachments = len(attachments)
countAttachment = 0
for attachment in attachments:
logger.debug('storeAttachmentResource: resId:%r collId:%r '
'content(size):%r fileFormat:%r filename:%r', resId,
collId, len(attachment['content']),
attachment['fileFormat'], attachment['filename'])
countAttachment += 1
if maarch_id:
email_to = notification_email if notification_email and countAttachment == nbAttachments else ''
logger.debug('storeAdditionalAttachmentResource: INES maarch_id: %r', maarch_id)
client.service.storeAdditionalAttachmentResource(
maarch_id,
resId, collId, attachment['content'],
attachment['fileFormat'], attachment['filename'], email_to)
else:
results = client.service.storeAttachmentResource(
resId, collId, attachment['content'],
attachment['fileFormat'], attachment['filename'])
res = sudsobject_to_dict(results)
debug_output['res'].append(res)
if debug:
data['debug'] = debug_output
logger.debug('debug : %r', data['debug'])
logger.debug('storeAttachment: resId:%r -- end', resId)
return utils.response_for_json(request, data)
class DebugView(DetailView):
@method_decorator(csrf_exempt)
def dispatch(self, *args, **kwargs):
return super(DebugView, self).dispatch(*args, **kwargs)
@utils.protected_api('can_access')
def post(self, request, *args, **kwargs):
client = self.get_client()
formdata = json.loads(request.body)
extras = formdata.get('extra', {})
logger.debug('formdata %r: ', formdata)
debug = False # 'debug' in request.GET
if debug:
debug_output = {}
# get formdef schema from wcs API
url = formdata['url']
p = urlparse.urlsplit(url)
scheme, netloc, path, query, fragment = \
p.scheme, p.netloc, p.path, p.query, p.fragment
schema_path = path.rsplit('/', 2)[0] + '/schema'
schema_url = urlparse.urlunsplit((scheme, netloc, schema_path, query, fragment))
schema = requests.get(schema_url).json()
# storeAttachmentResource attachments: list, build from formdata file fields
attachments = []
# regexp to rename attachments
regexp_filename_search_pattern_in_label, filename_result_pattern = self.prepare_regexp_filename_replacement(
extras)
#
# build document (encodedFile and fileFormat)
#
translations_txt = extras.get('translations')
if translations_txt:
# config = json.loads(open('config/defaults.json').read())
self.translations = json.loads(translations_txt)
logger.debug('Translation JSON: %r', self.translations)
else:
logger.debug('AUCUNE Translation JSON: %r', self.translations)
document = u'<html>'
document += u'<head><meta charset="utf-8"></head>'
document += '<body>'
document += u'<h1>%s</h1>' % self.translate('form_title', schema['name'])
page = ''
empty_page = True
for field in schema['fields']:
logger.debug('field type %r: ', field['type'])
if field['type'] == 'page':
# add last page, if it contains values
if page and not empty_page:
document += page
page = u'<hr /><h2>%s</h2>' % self.translate(field['label'], field['label'])
empty_page = True
# elif field['type'] == 'title':
# page += u'<h3>%s</h3>' % self.translate(field['label'], field['label'])
elif field['type'] == 'subtitle':
page += u'<h3>%s</h3>' % self.translate(field['label'], field['label'])
elif 'varname' in field:
varname = field['varname']
value = formdata['fields'].get(varname)
if not value:
continue
if field['type'] == 'file':
# field is a file: add it to attachments list
value['fileFormat'] = value['content_type'].split('/')[1] # FIXME (how ?)
if regexp_filename_search_pattern_in_label:
m = regexp_filename_search_pattern_in_label.match(field['label'])
if m:
value['filename_original'] = value['filename']
value['filename'] = string.replace(filename_result_pattern, 'searchedPatternInLabel',
m.group(1))
if 'filename' in filename_result_pattern:
value['filename'] = string.replace(value['filename'], 'filename',
value['filename_original'])
attachments.append(value)
value = '%s' % value['filename']
elif field['type'] == 'date':
value = datetime.strptime(value, '%Y-%m-%d')
value = value.strftime('%d-%m-%Y')
elif field['type'] == 'item' and self.translations:
value_raw = formdata['fields'].get(varname + '_raw')
value = self.translate(varname + '_' + value_raw, value)
page += u'<dl>'
page += u'<dt>%s</dt>' % self.translate(varname, field['label'])
page += u'<dd>%s</dd>' % value
page += u'</dl>'
empty_page = False
if page and not empty_page: # add last page, if it contains values
document += page
document += u'</body></html>'
logger.debug('DOCUMENT HTML %r ', document)
encodedFile = document.encode('utf-8').encode('base64')
fileFormat = 'html'
if debug:
debug_output['document'] = document
#
# build metadata for storeResource and storeExtResource
#
# storeResource metadata
metadata = {}
# storeExtResource metadata
ext_metadata = {}
# createContact data
contact = {}
contact_table1_pp = {'title', 'lastname', 'firstname', 'contact_type', 'is_corporate_person'}
# extract metadata and ext_metadata from formdata['extra']
for name, value in extras.items():
if name.startswith('maarch_contact_'):
contact[name[15:]] = value
if name.startswith('maarch_metadata_'):
metadata[name[16:]] = value
if name.startswith('maarch_ext_metadata_'):
ext_metadata[name[20:]] = value
def prepare_soap_contact(contact, contact_table1_pp):
datas = []
for name, value in contact.items():
data = client.factory.create('arrayOfDataContactContent')
data.column = name
if name.endswith('_date'):
data.type = 'date'
value = datetime.strptime(value[:19], '%Y-%m-%dT%H:%M:%S')
value = value.strftime('%d-%m-%Y %H:%M:%S')
elif isinstance(value, basestring):
data.type = 'string'
elif isinstance(value, int):
data.type = 'integer'
elif isinstance(value, float):
data.type = 'float'
elif value is None:
data.type = 'string'
value = ''
data.value = value
if name in contact_table1_pp:
data.table = 'contacts_v2'
else:
data.table = 'contact_addresses'
datas.append(data)
soap_contact = client.factory.create('arrayOfDataContact')
soap_contact.datas = datas
return soap_contact
# prepare metadata for SOAP call
def prepare_soap_metadata(metadata):
datas = []
for name, value in metadata.items():
data = client.factory.create('arrayOfDataContent')
data.column = name
if name.endswith('_date'):
data.type = 'date'
value = datetime.strptime(value[:19], '%Y-%m-%dT%H:%M:%S')
value = value.strftime('%d-%m-%Y %H:%M:%S')
elif isinstance(value, basestring):
data.type = 'string'
elif isinstance(value, int):
data.type = 'int'
elif isinstance(value, float):
data.type = 'float'
elif value is None:
data.type = 'string'
value = ''
data.value = value
datas.append(data)
soap_metadata = client.factory.create('arrayOfData')
soap_metadata.datas = datas
return soap_metadata
# logger.debug('Extraction DATA contact %r: ', contact)
# logger.debug('Extraction DATA metadata %r: ', metadata)
# if INES ESB (Tibco) between passerelle and Maarch, add a "maarch_id"
# parameter. Get value from formdata or workflow options.
maarch_id = extras.get('maarch_id')
if maarch_id:
contact = False
else:
contact = prepare_soap_contact(contact, contact_table1_pp)
metadata = prepare_soap_metadata(metadata)
if debug:
debug_output['contact'] = '%r' % contact
debug_output['metadata'] = '%r' % metadata
#
# get other Maarch variables (letterbox configuration by default)
#
collId = extras.get('maarch_collId') or 'letterbox_coll'
table = extras.get('maarch_table') or 'res_letterbox'
status = extras.get('maarch_status') or 'ATT'
ext_table = extras.get('maarch_ext_table') or 'mlb_coll_ext'
if debug:
debug_output['collId'] = collId
debug_output['table'] = table
debug_output['status'] = status
debug_output['ext_table'] = ext_table
if debug:
debug_output['maarch_id'] = maarch_id
#
# call Maarch web services
#
logger.debug('createContact: start')
if maarch_id:
logger.debug('createContact : SKIPPED done by INES for maarch_id: %r', maarch_id)
data_contact = False
else:
try:
# create_contact_result = client.service.CreateContact(contact)
# data_contact = sudsobject_to_dict(create_contact_result)
data_contact = False
logger.debug('createContact result: %r', data_contact)
except TypeNotFound:
pass
logger.debug('createContact: end')
logger.debug('storeResource+Ext+Attachment: start')
# Prepare ext_metadata
if data_contact:
ext_metadata['exp_contact_id'] = data_contact['contactId']
ext_metadata['address_id'] = data_contact['addressId']
ext_metadata = prepare_soap_metadata(ext_metadata)
# logger.debug('Extraction DATA ext_metadata %r: ', ext_metadata)
if debug:
debug_output['ext_metadata'] = '%r' % ext_metadata
# store the resource (storeResource)
logger.debug('storeResource: encodedFile(size):%r fileFormat:%r '
'collId:%r table:%r status:%r', len(encodedFile),
fileFormat, collId, table, status)
# logger.debug('storeResource: metadata: %r', metadata)
if maarch_id:
logger.debug('storeResource: INES maarch_id: %r', maarch_id)
# results = client.service.storeResource(
# maarch_id,
# encodedFile, metadata, collId,
# table, fileFormat, status)
# else:
# results = client.service.storeResource(
# encodedFile, metadata, collId,
# table, fileFormat, status)
# results = {'err': 0, 'data': {'returnCode': 0, 'resId': 1000}}
# data = sudsobject_to_dict(results)
# logger.debug('storeResource result: %r', data)
resId = 1000 # data.get('resId')
if not resId:
raise MaarchException('no resId after storeResource')
logger.debug('storeResource result: resId:%r', resId)
# logger.debug('storeExtResource: resId:%r ext_table:%r', resId,
# ext_table)
# logger.debug('storeExtResource: ext_metadata: %r', ext_metadata)
# store external metadata (storeExtResource)
if maarch_id:
logger.debug('storeExtResource: INES maarch_id: %r', maarch_id)
# results = client.service.storeExtResource(
# maarch_id,
# resId, ext_metadata, ext_table)
# else:
# results = client.service.storeExtResource(
# resId, ext_metadata, ext_table)
results = {'returnCode': 0, 'resId': 1000}
# store attachments
for attachment in attachments:
logger.debug('storeAttachmentResource: resId:%r collId:%r content(size):%r fileFormat:%r filename:%r',
resId, collId, len(attachment['content']), attachment['fileFormat'], attachment['filename'])
if maarch_id:
logger.debug('storeAttachmentResource: INES maarch_id: %r', maarch_id)
# client.service.storeAttachmentResource(
# maarch_id,
# resId, collId, attachment['content'],
# attachment['fileFormat'], attachment['filename'])
# else:
# client.service.storeAttachmentResource(
# resId, collId, attachment['content'],
# attachment['fileFormat'], attachment['filename'])
data = {}
data['resId'] = 1000
if debug:
data['debug'] = debug_output
logger.debug('storeResource+Ext+Attachment: resId:%r -- end', resId)
return utils.response_for_json(request, data)