wcs/wcs/admin/data_sources.py

639 lines
24 KiB
Python

# w.c.s. - web application for online forms
# Copyright (C) 2005-2015 Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
from quixote import get_publisher, get_request, redirect
from quixote.directory import Directory
from quixote.html import TemplateIO, htmltext
from wcs.admin import utils
from wcs.admin.categories import DataSourceCategoriesDirectory, get_categories
from wcs.backoffice.snapshots import SnapshotsDirectory
from wcs.carddef import CardDef
from wcs.categories import DataSourceCategory
from wcs.data_sources import (
DataSourceSelectionWidget,
NamedDataSource,
RefreshAgendas,
get_structured_items,
has_chrono,
)
from wcs.formdef import get_formdefs_of_all_kinds
from wcs.qommon import _, errors, misc, pgettext, template
from wcs.qommon.backoffice.menu import html_top
from wcs.qommon.errors import AccessForbiddenError
from wcs.qommon.form import (
CheckboxWidget,
ComputedExpressionWidget,
DurationWidget,
FileWidget,
Form,
HtmlWidget,
SingleSelectWidget,
SlugWidget,
StringWidget,
TextWidget,
WidgetDict,
WidgetList,
get_response,
get_session,
)
from wcs.roles import get_user_roles
class NamedDataSourceUI:
def __init__(self, datasource):
self.datasource = datasource
if self.datasource is None:
self.datasource = NamedDataSource()
def get_form(self):
form = Form(enctype='multipart/form-data', use_tabs=True)
form.add(StringWidget, 'name', title=_('Name'), required=True, size=30, value=self.datasource.name)
category_options = get_categories(DataSourceCategory)
if category_options and (not self.datasource or self.datasource.type != 'wcs:users'):
category_options = [(None, '---', '')] + list(category_options)
form.add(
SingleSelectWidget,
'category_id',
title=_('Category'),
options=category_options,
value=self.datasource.category_id,
)
form.add(
TextWidget,
'description',
title=_('Description'),
cols=40,
rows=5,
value=self.datasource.description,
)
if not self.datasource or (
self.datasource.type != 'wcs:users' and self.datasource.external != 'agenda_manual'
):
form.add(
DataSourceSelectionWidget,
'data_source',
value=self.datasource.data_source,
title=_('Data Source'),
allow_geojson=True,
allow_named_sources=False,
required=True,
)
form.add(
DurationWidget,
'cache_duration',
value=self.datasource.cache_duration,
title=_('Cache Duration'),
hint=_(
'Caching data will improve performances but will keep changes '
'from being visible immediately. You should keep this duration '
'reasonably short.'
),
required=False,
advanced=True,
attrs={
'data-dynamic-display-child-of': 'data_source$type',
'data-dynamic-display-value-in': 'json|geojson',
},
)
if not self.datasource or (
self.datasource.type != 'wcs:users' and self.datasource.external != 'agenda_manual'
):
form.add(
StringWidget,
'query_parameter',
value=self.datasource.query_parameter,
title=_('Query Parameter'),
hint=_('Name of the parameter to use for querying source (typically, q)'),
required=False,
advanced=True,
attrs={
'data-dynamic-display-child-of': 'data_source$type',
'data-dynamic-display-value': 'json',
},
)
form.add(
StringWidget,
'id_parameter',
value=self.datasource.id_parameter,
title=_('Id Parameter'),
hint=_('Name of the parameter to use to get a given entry from data source (typically, id)'),
required=False,
advanced=True,
attrs={
'data-dynamic-display-child-of': 'data_source$type',
'data-dynamic-display-value': 'json',
},
)
form.add(
StringWidget,
'id_property',
value=self.datasource.id_property,
title=_('Id Property'),
hint=_('Name of the property to use to get a given entry from data source (default: id)'),
required=False,
advanced=True,
attrs={
'data-dynamic-display-child-of': 'data_source$type',
'data-dynamic-display-value': 'geojson',
},
)
form.add(
StringWidget,
'label_template_property',
value=self.datasource.label_template_property,
title=_('Label template'),
hint=_('Django expression to build label of each value (default: {{ text }})'),
required=False,
advanced=True,
size=80,
attrs={
'data-dynamic-display-child-of': 'data_source$type',
'data-dynamic-display-value': 'geojson',
},
)
if self.datasource and self.datasource.type == 'wcs:users':
options = [(None, '---', None)]
options += get_user_roles()
form.add(
WidgetList,
'users_included_roles',
element_type=SingleSelectWidget,
value=self.datasource.users_included_roles,
title=_('Users with roles'),
add_element_label=_('Add Role'),
element_kwargs={'render_br': False, 'options': options},
)
form.add(
WidgetList,
'users_excluded_roles',
element_type=SingleSelectWidget,
value=self.datasource.users_excluded_roles,
title=_('Users without roles'),
add_element_label=_('Add Role'),
element_kwargs={'render_br': False, 'options': options},
)
form.add(
CheckboxWidget,
'include_disabled_users',
title=_('Include disabled users'),
value=self.datasource.include_disabled_users,
)
if self.datasource.slug and not self.datasource.is_used():
form.add(
SlugWidget,
'slug',
value=self.datasource.slug,
advanced=True,
)
if not self.datasource or self.datasource.type != 'wcs:users':
form.add(
StringWidget,
'data_attribute',
value=self.datasource.data_attribute,
title=_('Data Attribute'),
hint=_(
'Name of the attribute containing the list of results (default: data). '
'Possibility to chain attributes with a dot separator (example: data.results)'
),
required=False,
advanced=True,
attrs={
'data-dynamic-display-child-of': 'data_source$type',
'data-dynamic-display-value': 'json',
},
)
form.add(
StringWidget,
'id_attribute',
value=self.datasource.id_attribute,
title=_('Id Attribute'),
hint=_('Name of the attribute containing the identifier of an entry (default: id)'),
required=False,
advanced=True,
attrs={
'data-dynamic-display-child-of': 'data_source$type',
'data-dynamic-display-value': 'json',
},
)
form.add(
StringWidget,
'text_attribute',
value=self.datasource.text_attribute,
title=_('Text Attribute'),
hint=_('Name of the attribute containing the label of an entry (default: text)'),
required=False,
advanced=True,
attrs={
'data-dynamic-display-child-of': 'data_source$type',
'data-dynamic-display-value': 'json',
},
)
if not self.datasource or self.datasource.type != 'wcs:users':
form.add(
CheckboxWidget,
'notify_on_errors',
title=_('Notify on errors'),
value=self.datasource.notify_on_errors,
)
form.add(
CheckboxWidget,
'record_on_errors',
title=_('Record on errors'),
value=self.datasource.record_on_errors,
)
if self.datasource.external == 'agenda_manual':
form.add(
WidgetDict,
'qs_data',
title=_('Query string data'),
value=self.datasource.qs_data or {},
element_value_type=ComputedExpressionWidget,
element_value_kwargs={'allow_python': False},
allow_empty_values=True,
value_for_empty_value='',
)
if not self.datasource.is_readonly():
form.add_submit('submit', _('Submit'))
form.add_submit('cancel', _('Cancel'))
return form
def submit_form(self, form):
name = form.get_widget('name').parse()
slug_widget = form.get_widget('slug')
if slug_widget:
slug = form.get_widget('slug').parse()
for nds in NamedDataSource.select():
if nds.id == self.datasource.id:
continue
if name == nds.name:
form.get_widget('name').set_error(_('This name is already used.'))
if slug_widget and slug == nds.slug:
slug_widget.set_error(_('This value is already used.'))
if form.has_errors():
raise ValueError()
self.datasource.name = name
if slug_widget:
self.datasource.slug = slug
for widget in form.widgets:
if widget.name in ('name', 'slug'):
continue
setattr(self.datasource, widget.name, widget.parse())
self.datasource.store()
class NamedDataSourcePage(Directory):
_q_exports = [
'',
'edit',
'delete',
'export',
'duplicate',
('history', 'snapshots_dir'),
]
do_not_call_in_templates = True
def __init__(self, component=None, instance=None):
try:
self.datasource = instance or NamedDataSource.get(component)
except KeyError:
raise errors.TraversalError()
if self.datasource.external == 'agenda':
self.datasource.readonly = True
self.datasource_ui = NamedDataSourceUI(self.datasource)
get_response().breadcrumb.append((component + '/', self.datasource.name))
self.snapshots_dir = SnapshotsDirectory(self.datasource)
def get_sidebar(self):
r = TemplateIO(html=True)
if self.datasource.is_readonly():
r += htmltext('<div class="infonotice"><p>%s</p></div>') % _('This data source is readonly.')
if hasattr(self.datasource, 'snapshot_object'):
r += utils.snapshot_info_block(snapshot=self.datasource.snapshot_object)
r += htmltext('<ul id="sidebar-actions">')
if self.datasource.external == 'agenda' or not self.datasource.is_readonly():
r += htmltext('<li><a href="duplicate" rel="popup">%s</a></li>') % _('Duplicate')
if not self.datasource.is_readonly():
r += htmltext('<li><a href="delete" rel="popup">%s</a></li>') % _('Delete')
r += htmltext('<li><a href="export">%s</a></li>') % _('Export')
if get_publisher().snapshot_class:
r += htmltext('<li><a rel="popup" href="history/save">%s</a></li>') % _('Save snapshot')
r += htmltext('<li><a href="history/">%s</a></li>') % _('History')
r += htmltext('</ul>')
return r.getvalue()
def _q_index(self):
html_top('datasources', title=self.datasource.name)
get_response().filter['sidebar'] = self.get_sidebar()
url = None
if self.datasource.data_source and self.datasource.data_source.get('type') in (
'json',
'jsonp',
'geojson',
):
try:
url = self.datasource.get_variadic_url()
except Exception as exc:
url = '#%s' % exc
return template.QommonTemplateResponse(
templates=['wcs/backoffice/data-source.html'],
context={'view': self, 'datasource': self.datasource, 'roles': get_user_roles(), 'url': url},
)
def usage_in_formdefs(self):
formdefs = []
for formdef in get_formdefs_of_all_kinds():
if self.datasource.is_used_in_formdef(formdef):
formdefs.append(formdef)
formdefs.sort(key=lambda x: x.name.lower())
return formdefs
def preview_block(self):
try:
get_request().disable_error_notifications = True
data_source = self.datasource.extended_data_source
finally:
get_request().disable_error_notifications = False
if data_source.get('type') not in ('json', 'geojson', 'formula', 'wcs:users'):
return ''
try:
items = get_structured_items(data_source)
except Exception as exc:
return htmltext('<div class="warningnotice">%s (%r)</div>') % (
_('Unexpected fatal error getting items for preview.'),
exc,
)
if not items:
return ''
r = TemplateIO(html=True)
r += htmltext('<ul>')
additional_keys = set()
for item in items[:10]:
if not isinstance(item.get('text'), str):
r += htmltext('<li><tt>%s</tt>: <i>%s (%r)</i></li>') % (
item.get('id'),
_('error: not a string'),
item.get('text'),
)
else:
r += htmltext('<li><tt>%s</tt>: %s</li>') % (item.get('id'), item.get('text'))
if data_source.get('type') == 'geojson':
additional_keys.add('geometry_coordinates')
additional_keys.add('geometry_type')
additional_keys |= {'properties_%s' % k for k in item.get('properties', {}).keys()}
else:
additional_keys |= set(item.keys())
if len(items) > 10:
r += htmltext('<li>...</li>')
r += htmltext('</ul>')
additional_keys -= {'id', 'text', 'properties_id', 'properties_text'}
if additional_keys:
r += htmltext('<p>%s %s</p>') % (
_('Additional keys are available:'),
', '.join(sorted(additional_keys)),
)
return r.getvalue()
def edit(self):
form = self.datasource_ui.get_form()
if form.get_submit() == 'cancel':
return redirect('.')
if form.get_submit() == 'submit' and not form.has_errors():
try:
self.datasource_ui.submit_form(form)
except ValueError:
pass
else:
return redirect('.')
get_response().breadcrumb.append(('edit', _('Edit')))
html_top('datasources', title=_('Edit Data Source'))
r = TemplateIO(html=True)
r += htmltext('<h2>%s</h2>') % _('Edit Data Source')
r += form.render()
return r.getvalue()
def delete(self):
form = Form(enctype='multipart/form-data')
if not self.datasource.is_used():
form.widgets.append(
HtmlWidget('<p>%s</p>' % _('You are about to irrevocably delete this data source.'))
)
form.add_submit('delete', _('Submit'))
else:
form.widgets.append(
HtmlWidget('<p>%s</p>' % _('This datasource is still used, it cannot be deleted.'))
)
form.add_submit('cancel', _('Cancel'))
if form.get_widget('cancel').parse():
return redirect('..')
if not form.is_submitted() or form.has_errors():
get_response().breadcrumb.append(('delete', _('Delete')))
html_top('datasources', title=_('Delete Data Source'))
r = TemplateIO(html=True)
r += htmltext('<h2>%s %s</h2>') % (_('Deleting Data Source:'), self.datasource.name)
r += form.render()
return r.getvalue()
else:
self.datasource.remove_self()
return redirect('..')
def export(self):
return misc.xml_response(
self.datasource,
filename='datasource-%s.wcs' % self.datasource.slug,
content_type='application/x-wcs-datasource',
)
def duplicate(self):
if hasattr(self.datasource, 'snapshot_object'):
return redirect('.')
form = Form(enctype='multipart/form-data')
form.add_submit('duplicate', _('Submit'))
form.add_submit('cancel', _('Cancel'))
if form.get_widget('cancel').parse():
return redirect('.')
if not form.is_submitted() or form.has_errors():
get_response().breadcrumb.append(('duplicate', _('Duplicate')))
html_top('datasources', title=_('Duplicate Data Source'))
r = TemplateIO(html=True)
r += htmltext('<h2>%s %s</h2>') % (_('Duplicating Data Source:'), self.datasource.name)
r += form.render()
return r.getvalue()
tree = self.datasource.export_to_xml(include_id=True)
new_datasource = NamedDataSource.import_from_xml_tree(tree)
new_datasource.name = _('Copy of %s') % new_datasource.name
new_datasource.slug = new_datasource.get_new_slug(new_datasource.slug)
if self.datasource.agenda_ds:
new_datasource.external = 'agenda_manual'
new_datasource.store()
return redirect('../%s' % new_datasource.id)
class NamedDataSourcesDirectory(Directory):
_q_exports = [
'',
'new',
('new-users', 'new_users'),
'categories',
('import', 'p_import'),
('sync-agendas', 'sync_agendas'),
]
categories = DataSourceCategoriesDirectory()
def _q_traverse(self, path):
if (
not get_publisher().get_backoffice_root().is_global_accessible('forms')
and not get_publisher().get_backoffice_root().is_global_accessible('workflows')
and not get_publisher().get_backoffice_root().is_global_accessible('cards')
):
raise AccessForbiddenError()
get_response().breadcrumb.append(('data-sources/', _('Data Sources')))
return super()._q_traverse(path)
def _q_index(self):
html_top('datasources', title=_('Data Sources'))
data_sources = []
user_data_sources = []
agenda_data_sources = []
for ds in NamedDataSource.select(order_by='name'):
if ds.external == 'agenda':
agenda_data_sources.append(ds)
elif ds.type == 'wcs:users':
user_data_sources.append(ds)
else:
data_sources.append(ds)
categories = DataSourceCategory.select()
DataSourceCategory.sort_by_position(categories)
if categories:
categories.append(DataSourceCategory(pgettext('categories', 'Uncategorised')))
for category in categories:
category.data_sources = [x for x in data_sources if x.category_id == category.id]
generated_data_sources = list(CardDef.get_carddefs_as_data_source())
generated_data_sources.sort(key=lambda x: misc.simplify(x[1]))
return template.QommonTemplateResponse(
templates=['wcs/backoffice/data-sources.html'],
context={
'data_sources': data_sources,
'categories': categories,
'user_data_sources': user_data_sources,
'has_chrono': has_chrono(get_publisher()),
'has_users': True,
'agenda_data_sources': agenda_data_sources,
'generated_data_sources': generated_data_sources,
},
)
def _new(self, url, breadcrumb, title, ds_type=None):
get_response().breadcrumb.append((url, breadcrumb))
datasource = NamedDataSource()
if ds_type is not None:
datasource.data_source = {'type': ds_type}
datasource_ui = NamedDataSourceUI(datasource)
form = datasource_ui.get_form()
if form.get_widget('cancel').parse():
return redirect('.')
if form.get_submit() == 'submit' and not form.has_errors():
try:
datasource_ui.submit_form(form)
except ValueError:
pass
else:
return redirect('.')
html_top('datasources', title=title)
r = TemplateIO(html=True)
r += htmltext('<h2>%s</h2>') % title
r += form.render()
return r.getvalue()
def new(self):
return self._new(url='new', breadcrumb=_('New'), title=_('New Data Source'))
def new_users(self):
return self._new(
url='new-users',
breadcrumb=_('New Users Data Source'),
title=_('New Users Data Source'),
ds_type='wcs:users',
)
def _q_lookup(self, component):
return NamedDataSourcePage(component)
def p_import(self):
form = Form(enctype='multipart/form-data')
import_title = _('Import Data Source')
form.add(FileWidget, 'file', title=_('File'), required=True)
form.add_submit('submit', import_title)
form.add_submit('cancel', _('Cancel'))
if form.get_submit() == 'cancel':
return redirect('.')
if form.is_submitted() and not form.has_errors():
try:
return self.import_submit(form)
except ValueError:
pass
get_response().breadcrumb.append(('import', _('Import')))
html_top('datasources', title=import_title)
r = TemplateIO(html=True)
r += htmltext('<h2>%s</h2>') % import_title
r += htmltext('<p>%s</p>') % _('You can install a new data source by uploading a file.')
r += form.render()
return r.getvalue()
def import_submit(self, form):
fp = form.get_widget('file').parse().fp
error = False
try:
datasource = NamedDataSource.import_from_xml(fp)
get_session().message = ('info', _('This datasource has been successfully imported.'))
except ValueError:
error = True
if error:
form.set_error('file', _('Invalid File'))
raise ValueError()
try:
# check slug unicity
NamedDataSource.get_on_index(datasource.slug, 'slug', ignore_migration=True)
except KeyError:
pass
else:
datasource.slug = None # a new one will be set in .store()
datasource.store()
return redirect('%s/' % datasource.id)
def sync_agendas(self):
get_response().add_after_job(RefreshAgendas())
get_session().message = ('info', _('Agendas will be updated in the background.'))
return redirect('.')