Reutilisation de l'interface Web de requete de csvdatasource
This commit is contained in:
parent
b348d0e9f3
commit
429cc1d5f5
|
|
@ -36,36 +36,36 @@ class QueryForm(forms.ModelForm):
|
|||
}
|
||||
fields = '__all__'
|
||||
|
||||
#def clean_lines_of_expressions(self, lines, named=False):
|
||||
# if not lines:
|
||||
# return lines
|
||||
# errors = []
|
||||
# for i, line in enumerate(lines.splitlines()):
|
||||
# if named:
|
||||
# line = line.split(':', 1)
|
||||
# if len(line) != 2:
|
||||
# errors.append(ValidationError(
|
||||
# 'Syntax error line %d: each line must be prefixed '
|
||||
# 'with an identifier followed by a colon.' % (i + 1)))
|
||||
# continue
|
||||
# name, line = line
|
||||
# if not identifier_re.match(name):
|
||||
# errors.append(
|
||||
# ValidationError('Syntax error line %d: invalid identifier, '
|
||||
# 'it must starts with a letter and only '
|
||||
# 'contains letters, digits and _.' % (i + 1)))
|
||||
# continue
|
||||
# try:
|
||||
# get_code(line)
|
||||
# except SyntaxError as e:
|
||||
# errors.append(ValidationError(
|
||||
# 'Syntax error line %(line)d at character %(character)d' % {
|
||||
# 'line': i + 1,
|
||||
# 'character': e.offset
|
||||
# }))
|
||||
# if errors:
|
||||
# raise ValidationError(errors)
|
||||
# return lines
|
||||
def clean_lines_of_expressions(self, lines, named=False):
|
||||
if not lines:
|
||||
return lines
|
||||
errors = []
|
||||
for i, line in enumerate(lines.splitlines()):
|
||||
if named:
|
||||
line = line.split(':', 1)
|
||||
if len(line) != 2:
|
||||
errors.append(ValidationError(
|
||||
'Syntax error line %d: each line must be prefixed '
|
||||
'with an identifier followed by a colon.' % (i + 1)))
|
||||
continue
|
||||
name, line = line
|
||||
if not identifier_re.match(name):
|
||||
errors.append(
|
||||
ValidationError('Syntax error line %d: invalid identifier, '
|
||||
'it must starts with a letter and only '
|
||||
'contains letters, digits and _.' % (i + 1)))
|
||||
continue
|
||||
try:
|
||||
get_code(line)
|
||||
except SyntaxError as e:
|
||||
errors.append(ValidationError(
|
||||
'Syntax error line %(line)d at character %(character)d' % {
|
||||
'line': i + 1,
|
||||
'character': e.offset
|
||||
}))
|
||||
if errors:
|
||||
raise ValidationError(errors)
|
||||
return lines
|
||||
|
||||
def clean_filters(self):
|
||||
return self.clean_lines_of_expressions(self.data.get('filters'))
|
||||
|
|
|
|||
|
|
@ -28,6 +28,16 @@ def get_org_unit(u):
|
|||
return 0
|
||||
|
||||
class LDAPResource(BaseResource):
|
||||
ldif_file = models.FileField(_('LDIF File'), upload_to='ldif')
|
||||
#columns_keynames = models.CharField(
|
||||
# max_length=256,
|
||||
# verbose_name=_('Column keynames'),
|
||||
# default='id, text',
|
||||
# help_text=_('ex: id,text,data1,data2'), blank=True)
|
||||
#skip_header = models.BooleanField(_('Skip first line'), default=False)
|
||||
_dialect_options = jsonfield.JSONField(editable=False, null=True)
|
||||
#sheet_name = models.CharField(_('Sheet name'), blank=True, max_length=150)
|
||||
|
||||
category = 'Identity Management Connectors'
|
||||
class Meta:
|
||||
verbose_name = 'LDAP'
|
||||
|
|
@ -47,12 +57,265 @@ class LDAPResource(BaseResource):
|
|||
|
||||
@classmethod
|
||||
def get_verbose_name(cls):
|
||||
#return cls._meta.verbose_name
|
||||
return "LDAP Connector"
|
||||
|
||||
@classmethod
|
||||
def is_enabled(cls):
|
||||
return True
|
||||
|
||||
def clean(self, *args, **kwargs):
|
||||
file_type = self.ldif_file.name.split('.')[-1]
|
||||
#if file_type in ('ods', 'xls', 'xlsx') and not self.sheet_name:
|
||||
# raise ValidationError(_('You must specify a sheet name'))
|
||||
#return super(CsvDataSource, self).clean(*args, **kwargs)
|
||||
return "TODO"
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
file_type = self.ldif_file.name.split('.')[-1]
|
||||
#if file_type not in ('ods', 'xls', 'xlsx'):
|
||||
# content = self.get_content_without_bom()
|
||||
# dialect = csv.Sniffer().sniff(content)
|
||||
# self.dialect_options = {
|
||||
# k: v for k, v in vars(dialect).items() if not k.startswith('_')
|
||||
# }
|
||||
#return super(CsvDataSource, self).save(*args, **kwargs)
|
||||
return "TODO"
|
||||
|
||||
@property
|
||||
def dialect_options(self):
|
||||
"""turn dict items into string
|
||||
"""
|
||||
# Set dialect_options if None
|
||||
if self._dialect_options is None:
|
||||
self.save()
|
||||
|
||||
options = {}
|
||||
for k, v in self._dialect_options.items():
|
||||
if isinstance(v, unicode):
|
||||
v = v.encode('ascii')
|
||||
options[k.encode('ascii')] = v
|
||||
|
||||
return options
|
||||
|
||||
@dialect_options.setter
|
||||
def dialect_options(self, value):
|
||||
self._dialect_options = value
|
||||
|
||||
def get_content_without_bom(self):
|
||||
self.ldif_file.seek(0)
|
||||
content = self.ldif_file.read()
|
||||
return content.decode('utf-8-sig', 'ignore').encode('utf-8')
|
||||
|
||||
def get_rows(self):
|
||||
#file_type = self.csv_file.name.split('.')[-1]
|
||||
#if file_type not in ('ods', 'xls', 'xlsx'):
|
||||
# content = self.get_content_without_bom()
|
||||
# reader = csv.reader(content.splitlines(), **self.dialect_options)
|
||||
# rows = list(reader)
|
||||
|
||||
#else:
|
||||
# if file_type == 'ods':
|
||||
# content = get_data_ods(self.csv_file)
|
||||
|
||||
# elif file_type == 'xls' or file_type == 'xlsx':
|
||||
# content = get_data_xls(self.csv_file.path)
|
||||
|
||||
# if self.sheet_name not in content:
|
||||
# return []
|
||||
|
||||
# rows = content[self.sheet_name]
|
||||
|
||||
#if not rows:
|
||||
# return []
|
||||
|
||||
#if self.skip_header:
|
||||
# rows = rows[1:]
|
||||
|
||||
#return [[smart_text(x) for x in y] for y in rows]
|
||||
return "TODO"
|
||||
|
||||
def get_data(self, filters=None):
|
||||
titles = [t.strip() for t in self.columns_keynames.split(',')]
|
||||
indexes = [titles.index(t) for t in titles if t]
|
||||
caption = [titles[i] for i in indexes]
|
||||
|
||||
# validate filters (appropriate columns must exist)
|
||||
if filters:
|
||||
for filter_key in filters.keys():
|
||||
if not filter_key.split(lookups.DELIMITER)[0] in titles:
|
||||
del filters[filter_key]
|
||||
|
||||
rows = self.get_rows()
|
||||
|
||||
data = []
|
||||
|
||||
# build a generator of all filters
|
||||
def filters_generator(filters, titles):
|
||||
if not filters:
|
||||
return
|
||||
for key, value in filters.items():
|
||||
try:
|
||||
key, op = key.split(lookups.DELIMITER)
|
||||
except (ValueError,):
|
||||
op = 'eq'
|
||||
|
||||
index = titles.index(key)
|
||||
|
||||
yield lookups.get_lookup(op, index, value)
|
||||
|
||||
# apply filters to data
|
||||
def super_filter(filters, data):
|
||||
for f in filters:
|
||||
data = itertools.ifilter(f, data)
|
||||
return data
|
||||
|
||||
matches = super_filter(
|
||||
filters_generator(filters, titles), rows
|
||||
)
|
||||
|
||||
for row in matches:
|
||||
line = []
|
||||
for i in indexes:
|
||||
try:
|
||||
line.append(row[i])
|
||||
except IndexError:
|
||||
line.append('')
|
||||
|
||||
data.append(dict(zip(caption, line)))
|
||||
|
||||
return data
|
||||
|
||||
@property
|
||||
def titles(self):
|
||||
return [smart_text(t.strip()) for t in self.columns_keynames.split(',')]
|
||||
|
||||
@endpoint('json-api', perm='can_access', methods=['get'],
|
||||
name='query', pattern='^(?P<query_name>[\w-]+)/$')
|
||||
def select(self, request, query_name, **kwargs):
|
||||
try:
|
||||
query = Query.objects.get(resource=self.id, slug=query_name)
|
||||
except Query.DoesNotExist:
|
||||
raise APIError(u'no such query')
|
||||
|
||||
titles = self.titles
|
||||
rows = self.get_rows()
|
||||
data = [dict(zip(titles, line)) for line in rows]
|
||||
|
||||
def stream_expressions(expressions, data, kind, titles=None):
|
||||
codes = []
|
||||
for i, expr in enumerate(expressions):
|
||||
try:
|
||||
code = get_code(expr)
|
||||
except (TypeError, SyntaxError) as e:
|
||||
data = {
|
||||
'expr': expr,
|
||||
'error': unicode(e)
|
||||
}
|
||||
if titles:
|
||||
data['name'] = titles[i]
|
||||
else:
|
||||
data['idx'] = i
|
||||
raise APIError(u'invalid %s expression' % kind, data=data)
|
||||
codes.append((code, expr))
|
||||
for row in data:
|
||||
new_row = []
|
||||
row_vars = dict(row)
|
||||
row_vars['query'] = kwargs
|
||||
for i, (code, expr) in enumerate(codes):
|
||||
try:
|
||||
result = eval(code, {}, row_vars)
|
||||
except Exception as e:
|
||||
data = {
|
||||
'expr': expr,
|
||||
'row': repr(row),
|
||||
}
|
||||
if titles:
|
||||
data['name'] = titles[i]
|
||||
else:
|
||||
data['idx'] = i
|
||||
raise APIError(u'invalid %s expression' % kind, data=data)
|
||||
new_row.append(result)
|
||||
yield new_row, row
|
||||
|
||||
filters = query.get_list('filters')
|
||||
|
||||
if filters:
|
||||
data = [row for new_row, row in stream_expressions(filters, data, kind='filters')
|
||||
if all(new_row)]
|
||||
|
||||
order = query.get_list('order')
|
||||
if order:
|
||||
generator = stream_expressions(order, data, kind='order')
|
||||
new_data = [(tuple(new_row), row) for new_row, row in generator]
|
||||
new_data.sort(key=lambda (k, row): k)
|
||||
data = [row for key, row in new_data]
|
||||
|
||||
distinct = query.get_list('distinct')
|
||||
if distinct:
|
||||
generator = stream_expressions(distinct, data, kind='distinct')
|
||||
seen = set()
|
||||
new_data = []
|
||||
for new_row, row in generator:
|
||||
new_row = tuple(new_row)
|
||||
try:
|
||||
hash(new_row)
|
||||
except TypeError:
|
||||
raise APIError(u'distinct value is unhashable',
|
||||
data={
|
||||
'row': repr(row),
|
||||
'distinct': repr(new_row),
|
||||
})
|
||||
if new_row in seen:
|
||||
continue
|
||||
new_data.append(row)
|
||||
seen.add(new_row)
|
||||
data = new_data
|
||||
|
||||
projection = query.get_list('projections')
|
||||
if projection:
|
||||
expressions = []
|
||||
titles = []
|
||||
for mapping in projection:
|
||||
name, expr = mapping.split(':', 1)
|
||||
if not identifier_re.match(name):
|
||||
raise APIError(u'invalid projection name', data=name)
|
||||
titles.append(name)
|
||||
expressions.append(expr)
|
||||
new_data = []
|
||||
for new_row, row in stream_expressions(expressions, data, kind='projection',
|
||||
titles=titles):
|
||||
new_data.append(dict(zip(titles, new_row)))
|
||||
data = new_data
|
||||
|
||||
# allow jsonp queries by select2
|
||||
# filtering is done there afater projection because we need a projection named text for
|
||||
# retro-compatibility with previous use of the csvdatasource with select2
|
||||
if 'q' in request.GET:
|
||||
if 'case-insensitive' in request.GET:
|
||||
filters = ["query['q'].lower() in text.lower()"]
|
||||
else:
|
||||
filters = ["query['q'] in text"]
|
||||
data = [row for new_row, row in stream_expressions(filters, data, kind='filters')
|
||||
if new_row[0]]
|
||||
|
||||
if query.structure == 'array':
|
||||
return [[row[t] for t in titles] for row in data]
|
||||
elif query.structure == 'dict':
|
||||
return data
|
||||
elif query.structure == 'tuples':
|
||||
return [[[t, row[t]] for t in titles] for row in data]
|
||||
elif query.structure == 'onerow':
|
||||
if len(data) != 1:
|
||||
raise APIError('more or less than one row', data=data)
|
||||
return data[0]
|
||||
elif query.structure == 'one':
|
||||
if len(data) != 1:
|
||||
raise APIError('more or less than one row', data=data)
|
||||
if len(data[0]) != 1:
|
||||
raise APIError('more or less than one column', data=data)
|
||||
return data[0].values()[0]
|
||||
|
||||
class Query(models.Model):
|
||||
resource = models.ForeignKey('LDAPResource')
|
||||
slug = models.SlugField('Name (slug)')
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
<p>
|
||||
{% trans "File:" %}
|
||||
{% if object|can_edit:request.user %}<a href="{% url 'ldap-download' connector_slug=object.slug %}">{{object.ldap_file}}</a>
|
||||
{% else %}{{object.ldap_file}}{% endif %}
|
||||
{% if object|can_edit:request.user %}<a href="{% url 'ldap-download' connector_slug=object.slug %}">{{object.ldif_file}}</a>
|
||||
{% else %}{{object.ldif_file}}{% endif %}
|
||||
</p>
|
||||
|
||||
<div id="endpoints">
|
||||
|
|
|
|||
|
|
@ -84,5 +84,3 @@ def ldap_add_entry(id):
|
|||
ldap_terminate(l)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ from .forms import QueryForm
|
|||
|
||||
#TODO
|
||||
# derive csv connector
|
||||
# use ldap3 instead of python-ldap
|
||||
# online LDAP query
|
||||
# LDIF import
|
||||
|
||||
# Create your views here.
|
||||
def dummy_view(request):
|
||||
|
|
|
|||
Reference in New Issue