445 lines
18 KiB
Python
445 lines
18 KiB
Python
# w.c.s. - web application for online forms
|
|
# Copyright (C) 2005-2013 Entr'ouvert
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import datetime
|
|
import json
|
|
import sys
|
|
import traceback
|
|
import xml.etree.ElementTree as ET
|
|
import collections
|
|
import mimetypes
|
|
from StringIO import StringIO
|
|
import urllib
|
|
import urlparse
|
|
|
|
|
|
from quixote.html import TemplateIO, htmltext
|
|
from qommon.errors import ConnectionError
|
|
from qommon.form import *
|
|
from qommon.misc import (http_get_page, http_post_request, get_variadic_url,
|
|
JSONEncoder, json_loads, site_encode)
|
|
from wcs.workflows import (WorkflowStatusItem, register_item_class,
|
|
AbortActionException, AttachmentEvolutionPart)
|
|
from wcs.api_utils import sign_url
|
|
|
|
TIMEOUT = 30
|
|
|
|
class JournalWsCallErrorPart: #pylint: disable=C1001
|
|
content = None
|
|
data = None
|
|
label = None
|
|
|
|
def __init__(self, summary, label=None, data=None):
|
|
self.summary = summary
|
|
self.label = label
|
|
if data:
|
|
self.data = data[:10000] # beware of huge responses
|
|
|
|
def view(self):
|
|
if not get_request().get_path().startswith('/backoffice/'):
|
|
return ''
|
|
r = TemplateIO(html=True)
|
|
r += htmltext('<div class="ws-error">')
|
|
r += htmltext('<h4 class="foldable folded">')
|
|
if self.label:
|
|
r += _('Error during webservice call "%s"') % self.label
|
|
else:
|
|
r += _('Error during webservice call')
|
|
r += htmltext('</h4>')
|
|
r += htmltext('<div>')
|
|
r += htmltext('<p>%s</p>\n') % self.summary
|
|
if self.data:
|
|
try:
|
|
json_data = json.loads(self.data)
|
|
except ValueError:
|
|
pass
|
|
else:
|
|
labels = {
|
|
'err': _('Error Code'),
|
|
'err_class': _('Error Class'),
|
|
'err_desc': _('Error Description')
|
|
}
|
|
r += htmltext('<ul>')
|
|
for attr in ('err', 'err_class', 'err_desc'):
|
|
if attr in json_data:
|
|
r += htmltext('<li>%s: %s</li>\n' ) % (
|
|
labels.get(attr), site_encode(json_data[attr]))
|
|
r += htmltext('</ul>')
|
|
r += htmltext('</div>')
|
|
r += htmltext('</div>')
|
|
return r.getvalue()
|
|
|
|
def get_json_export_dict(self, anonymise=False):
|
|
d = {
|
|
'type': 'wscall-error',
|
|
}
|
|
if not anonymise:
|
|
d.update({
|
|
'summary': self.summary,
|
|
'label': self.label,
|
|
'data': self.data,
|
|
})
|
|
return d
|
|
|
|
|
|
class WebserviceCallStatusItem(WorkflowStatusItem):
|
|
description = N_('Webservice Call')
|
|
key = 'webservice_call'
|
|
support_substitution_variables = True
|
|
|
|
label = None
|
|
url = None
|
|
varname = None
|
|
post = True
|
|
request_signature_key = None
|
|
post_data = None
|
|
qs_data = None
|
|
_method = None
|
|
response_type = 'json'
|
|
|
|
action_on_4xx = ':stop'
|
|
action_on_5xx = ':stop'
|
|
action_on_bad_data = ':pass'
|
|
action_on_network_errors = ':stop'
|
|
notify_on_errors = True
|
|
record_errors = False
|
|
|
|
@property
|
|
def method(self):
|
|
if self._method in ('GET', 'POST'):
|
|
return self._method
|
|
if self.post or self.post_data:
|
|
return 'POST'
|
|
return 'GET'
|
|
|
|
@method.setter
|
|
def method(self, value):
|
|
self._method = value
|
|
|
|
def render_as_line(self):
|
|
if self.label:
|
|
return _('Webservice Call "%s"') % self.label
|
|
else:
|
|
return _('Webservice Call')
|
|
|
|
def get_parameters(self):
|
|
return ('url', 'post', 'varname', 'request_signature_key', 'post_data',
|
|
'action_on_4xx', 'action_on_5xx', 'action_on_bad_data',
|
|
'action_on_network_errors', 'notify_on_errors',
|
|
'record_errors', 'label', 'method', 'response_type',
|
|
'qs_data')
|
|
|
|
def add_parameters_widgets(self, form, parameters, prefix='', formdef=None):
|
|
if 'label' in parameters:
|
|
form.add(StringWidget, '%slabel' % prefix, size=40, title=_('Label'), value=self.label)
|
|
|
|
form.widgets.append(HtmlWidget(htmltext('<h3>%s</h3>') % _('Request')))
|
|
if 'url' in parameters:
|
|
form.add(StringWidget, '%surl' % prefix,
|
|
title=_('URL'), value=self.url, size=80,
|
|
hint=_('Common substitution variables are available with the [variable] syntax.'))
|
|
if 'request_signature_key' in parameters:
|
|
form.add(ComputedExpressionWidget, '%srequest_signature_key' % prefix,
|
|
title=_('Request Signature Key'),
|
|
value=self.request_signature_key)
|
|
if 'qs_data' in parameters:
|
|
form.add(WidgetDict, '%sqs_data' % prefix,
|
|
title=_('Query string data'),
|
|
value=self.qs_data or {},
|
|
element_value_type=ComputedExpressionWidget)
|
|
methods = collections.OrderedDict(
|
|
[('GET', _('GET')), ('POST', _('POST (JSON)'))])
|
|
if 'method' in parameters:
|
|
form.add(RadiobuttonsWidget, '%smethod' % prefix,
|
|
title=_('Method'),
|
|
options=methods.items(),
|
|
value=self.method,
|
|
attrs={'data-dynamic-display-parent': 'true'})
|
|
if 'post' in parameters:
|
|
form.add(CheckboxWidget, '%spost' % prefix,
|
|
title=_('Post formdata'),
|
|
value=self.post,
|
|
attrs={
|
|
'data-dynamic-display-child-of': '%smethod' % prefix,
|
|
'data-dynamic-display-value': methods.get('POST'),
|
|
})
|
|
if 'post_data' in parameters:
|
|
form.add(WidgetDict, '%spost_data' % prefix,
|
|
title=_('Post data'),
|
|
value=self.post_data or {},
|
|
element_value_type=ComputedExpressionWidget,
|
|
attrs={
|
|
'data-dynamic-display-child-of': '%smethod' % prefix,
|
|
'data-dynamic-display-value': methods.get('POST'),
|
|
})
|
|
|
|
form.widgets.append(HtmlWidget(htmltext('<h3>%s</h3>') % _('Response')))
|
|
response_types = collections.OrderedDict(
|
|
[('json', _('JSON')), ('attachment', _('Attachment'))])
|
|
if 'response_type' in parameters:
|
|
form.add(RadiobuttonsWidget, '%sresponse_type' % prefix,
|
|
title=_('Response Type'),
|
|
options=response_types.items(),
|
|
value=self.response_type,
|
|
attrs={'data-dynamic-display-parent': 'true'})
|
|
if 'varname' in parameters:
|
|
form.add(VarnameWidget, '%svarname' % prefix,
|
|
title=_('Variable Name'), value=self.varname)
|
|
|
|
form.widgets.append(HtmlWidget(htmltext('<h3>%s</h3>') % _('Error Handling')))
|
|
error_actions = [(':stop', _('Stop')), (':pass', _('Ignore'))]
|
|
error_actions.extend([(x.id, _('Jump to %s') % x.name) for x in
|
|
self.parent.parent.possible_status])
|
|
for attribute in ('action_on_4xx', 'action_on_5xx', 'action_on_network_errors',
|
|
'action_on_bad_data'):
|
|
if not attribute in parameters:
|
|
continue
|
|
if attribute == 'action_on_bad_data':
|
|
attrs = {
|
|
'data-dynamic-display-child-of': '%sresponse_type' % prefix,
|
|
'data-dynamic-display-value': response_types.get('json'),
|
|
}
|
|
else:
|
|
attrs = {}
|
|
label = {
|
|
'action_on_4xx': _('Action on HTTP error 4xx'),
|
|
'action_on_5xx': _('Action on HTTP error 5xx'),
|
|
'action_on_bad_data': _('Action on non-JSON response'),
|
|
'action_on_network_errors': _('Action on network errors')
|
|
}.get(attribute)
|
|
form.add(SingleSelectWidget, '%s%s' % (prefix, attribute),
|
|
title=label,
|
|
value=getattr(self, attribute),
|
|
options=error_actions, attrs=attrs)
|
|
|
|
if 'notify_on_errors' in parameters:
|
|
form.add(CheckboxWidget, '%snotify_on_errors' % prefix,
|
|
title=_('Notify on errors'),
|
|
value=self.notify_on_errors)
|
|
|
|
if 'record_errors' in parameters:
|
|
form.add(CheckboxWidget, '%srecord_errors' % prefix,
|
|
title=_('Record errors in the log'),
|
|
value=self.record_errors)
|
|
|
|
def perform(self, formdata):
|
|
if not self.url:
|
|
# misconfigured action
|
|
return
|
|
url = self.url
|
|
if '[' in url:
|
|
variables = get_publisher().substitutions.get_context_variables()
|
|
url = get_variadic_url(url, variables)
|
|
|
|
if self.qs_data: # merge qs_data into url
|
|
publisher = get_publisher()
|
|
parsed = urlparse.urlparse(url)
|
|
qs = list(urlparse.parse_qsl(parsed.query))
|
|
for key, value in self.qs_data.iteritems():
|
|
try:
|
|
value = self.compute(value, raises=True)
|
|
value = str(value)
|
|
except:
|
|
get_publisher().notify_of_exception(sys.exc_info())
|
|
else:
|
|
key = publisher.sitecharset2utf8(key)
|
|
value = publisher.sitecharset2utf8(value)
|
|
qs.append((key, value))
|
|
qs = urllib.urlencode(qs)
|
|
url = urlparse.urlunparse(parsed[:4] + (qs,) + parsed[5:6])
|
|
|
|
if self.request_signature_key:
|
|
signature_key = self.compute(self.request_signature_key)
|
|
if signature_key:
|
|
url = sign_url(url, signature_key)
|
|
|
|
headers = {'Content-type': 'application/json',
|
|
'Accept': 'application/json'}
|
|
post_data = None # payload
|
|
|
|
# if self.post_data exists, post_data is a dict built from it
|
|
if self.method == 'POST' and self.post_data:
|
|
post_data = {}
|
|
for (key, value) in self.post_data.items():
|
|
try:
|
|
post_data[key] = self.compute(value, raises=True)
|
|
except:
|
|
get_publisher().notify_of_exception(sys.exc_info())
|
|
|
|
# if formdata has to be sent, it's the payload. If post_data exists,
|
|
# it's added in formdata['extra']
|
|
if self.method == 'POST' and self.post:
|
|
formdata_dict = formdata.get_json_export_dict()
|
|
if post_data is not None:
|
|
formdata_dict['extra'] = post_data
|
|
post_data = formdata_dict
|
|
|
|
try:
|
|
if self.method == 'POST':
|
|
if post_data:
|
|
post_data = json.dumps(post_data, cls=JSONEncoder,
|
|
encoding=get_publisher().site_charset)
|
|
# increase timeout for huge loads, one second every 65536
|
|
# bytes, to match a country 512kbps DSL line.
|
|
timeout = TIMEOUT
|
|
timeout += len(post_data) / 65536
|
|
response, status, data, auth_header = http_post_request(
|
|
url, post_data, headers=headers, timeout=timeout)
|
|
else:
|
|
response, status, data, auth_header = http_get_page(
|
|
url, headers=headers, timeout=TIMEOUT)
|
|
except ConnectionError as e:
|
|
status = 0
|
|
self.action_on_error(self.action_on_network_errors, formdata,
|
|
exc_info=sys.exc_info())
|
|
|
|
if self.varname:
|
|
workflow_data = {
|
|
'%s_status' % self.varname: status,
|
|
'%s_time' % self.varname: datetime.datetime.now().isoformat(),
|
|
}
|
|
if status in (204, 205):
|
|
pass # not returning any content
|
|
elif (status // 100) == 2:
|
|
self.store_response(formdata, response, data, workflow_data)
|
|
else: # on error, record data if it is JSON
|
|
try:
|
|
d = json_loads(data)
|
|
except (ValueError, TypeError) as e:
|
|
pass
|
|
else:
|
|
workflow_data['%s_error_response' % self.varname] = d
|
|
formdata.update_workflow_data(workflow_data)
|
|
formdata.store()
|
|
|
|
if (status // 100) == 4:
|
|
self.action_on_error(self.action_on_4xx, formdata, response, data=data)
|
|
if (status // 100) == 5:
|
|
self.action_on_error(self.action_on_5xx, formdata, response, data=data)
|
|
|
|
def store_response(self, formdata, response, data, workflow_data):
|
|
if self.response_type == 'json':
|
|
try:
|
|
d = json_loads(data)
|
|
except (ValueError, TypeError) as e:
|
|
formdata.update_workflow_data(workflow_data)
|
|
formdata.store()
|
|
self.action_on_error(self.action_on_bad_data, formdata,
|
|
response, data=data, exc_info=sys.exc_info())
|
|
else:
|
|
workflow_data['%s_response' % self.varname] = d
|
|
if isinstance(d.get('data'), dict) and d['data'].get('display_id'):
|
|
formdata.id_display = d.get('data', {}).get('display_id')
|
|
elif d.get('display_id'):
|
|
formdata.id_display = d.get('display_id')
|
|
else: # store result as attachment
|
|
content_type = response.getheader('content-type') or ''
|
|
if content_type:
|
|
content_type = content_type.split(';')[0].strip().lower()
|
|
workflow_data['%s_content_type' % self.varname] = content_type
|
|
workflow_data['%s_length' % self.varname] = len(data)
|
|
extension = mimetypes.guess_extension(content_type, strict=False) or ''
|
|
filename = '%s%s' % (self.varname, extension)
|
|
fp_content = StringIO(data)
|
|
attachment = AttachmentEvolutionPart(filename, fp_content,
|
|
content_type=content_type,
|
|
varname=self.varname)
|
|
formdata.evolution[-1].add_part(attachment)
|
|
|
|
def action_on_error(self, action, formdata, response=None, data=None, exc_info=None):
|
|
if action in (':pass', ':stop') and (self.notify_on_errors or self.record_errors):
|
|
if exc_info:
|
|
summary = traceback.format_exception_only(exc_info[0], exc_info[1])[-1]
|
|
else:
|
|
summary = ''
|
|
if response:
|
|
summary = '%s %s' % (response.status, response.reason)
|
|
try:
|
|
raise Exception(summary)
|
|
except Exception as e:
|
|
exc_info = sys.exc_info()
|
|
|
|
if self.notify_on_errors:
|
|
get_publisher().notify_of_exception(exc_info, context='[WSCALL]')
|
|
if self.record_errors and formdata.evolution:
|
|
formdata.evolution[-1].add_part(
|
|
JournalWsCallErrorPart(summary, self.label, data))
|
|
formdata.store()
|
|
if action == ':pass':
|
|
return
|
|
if action == ':stop':
|
|
raise AbortActionException()
|
|
formdata.status = 'wf-%s' % action
|
|
formdata.store()
|
|
raise AbortActionException()
|
|
|
|
def get_target_status(self):
|
|
targets = []
|
|
for attribute in ('action_on_4xx', 'action_on_5xx', 'action_on_bad_data',
|
|
'action_on_network_errors'):
|
|
value = getattr(self, attribute)
|
|
if value in (':pass', ':stop'):
|
|
continue
|
|
targets.append(self.parent.parent.get_status(value))
|
|
return targets
|
|
|
|
def get_jump_label(self):
|
|
return _('Error calling webservice')
|
|
|
|
def _kv_data_export_to_xml(self, xml_item, charset, include_id, attribute):
|
|
assert attribute
|
|
if not getattr(self, attribute):
|
|
return
|
|
el = ET.SubElement(xml_item, attribute)
|
|
for (key, value) in getattr(self, attribute).items():
|
|
item = ET.SubElement(el, 'item')
|
|
if type(key) is unicode:
|
|
ET.SubElement(item, 'name').text = key
|
|
elif type(key) is str:
|
|
ET.SubElement(item, 'name').text = unicode(key, charset, 'replace')
|
|
else:
|
|
raise AssertionError('unknown type for key (%r)' % key)
|
|
if type(value) is unicode:
|
|
ET.SubElement(item, 'value').text = value
|
|
elif type(value) is str:
|
|
ET.SubElement(item, 'value').text = unicode(value, charset, 'replace')
|
|
else:
|
|
raise AssertionError('unknown type for value (%r)' % key)
|
|
|
|
def _kv_data_init_with_xml(self, elem, charset, include_id, attribute):
|
|
if elem is None:
|
|
return
|
|
setattr(self, attribute, {})
|
|
for item in elem.findall('item'):
|
|
key = item.find('name').text.encode(charset)
|
|
value = item.find('value').text.encode(charset)
|
|
getattr(self, attribute)[key] = value
|
|
|
|
def post_data_export_to_xml(self, xml_item, charset, include_id=False):
|
|
self._kv_data_export_to_xml(xml_item, charset, include_id=include_id,
|
|
attribute='post_data')
|
|
|
|
def post_data_init_with_xml(self, elem, charset, include_id=False):
|
|
self._kv_data_init_with_xml(elem, charset, include_id=include_id, attribute='post_data')
|
|
|
|
def qs_data_export_to_xml(self, xml_item, charset, include_id=False):
|
|
self._kv_data_export_to_xml(xml_item, charset, include_id=include_id, attribute='qs_data')
|
|
|
|
def qs_data_init_with_xml(self, elem, charset, include_id=False):
|
|
self._kv_data_init_with_xml(elem, charset, include_id=include_id, attribute='qs_data')
|
|
|
|
register_item_class(WebserviceCallStatusItem)
|