Retrait fonctions specifiques au format csv
This commit is contained in:
parent
429cc1d5f5
commit
f6664aede0
|
|
@ -1,11 +1,13 @@
|
|||
from django.core.urlresolvers import reverse
|
||||
from django.db import models
|
||||
from django.utils.encoding import smart_text
|
||||
#from django import forms
|
||||
from passerelle.base.models import BaseResource
|
||||
from passerelle.utils.jsonresponse import APIError
|
||||
from passerelle.utils.api import endpoint
|
||||
|
||||
import sys
|
||||
import jsonfield
|
||||
|
||||
from .utils import base
|
||||
|
||||
|
|
@ -28,14 +30,14 @@ def get_org_unit(u):
|
|||
return 0
|
||||
|
||||
class LDAPResource(BaseResource):
|
||||
ldif_file = models.FileField(_('LDIF File'), upload_to='ldif')
|
||||
ldif_file = models.FileField('LDIF File', upload_to='ldif')
|
||||
#columns_keynames = models.CharField(
|
||||
# max_length=256,
|
||||
# verbose_name=_('Column keynames'),
|
||||
# default='id, text',
|
||||
# help_text=_('ex: id,text,data1,data2'), blank=True)
|
||||
#skip_header = models.BooleanField(_('Skip first line'), default=False)
|
||||
_dialect_options = jsonfield.JSONField(editable=False, null=True)
|
||||
_dialect_options = jsonfield.JSONField(editable=False, null=True) #FIXME
|
||||
#sheet_name = models.CharField(_('Sheet name'), blank=True, max_length=150)
|
||||
|
||||
category = 'Identity Management Connectors'
|
||||
|
|
@ -65,25 +67,23 @@ class LDAPResource(BaseResource):
|
|||
return True
|
||||
|
||||
def clean(self, *args, **kwargs):
|
||||
file_type = self.ldif_file.name.split('.')[-1]
|
||||
#file_type = self.ldif_file.name.split('.')[-1]
|
||||
#if file_type in ('ods', 'xls', 'xlsx') and not self.sheet_name:
|
||||
# raise ValidationError(_('You must specify a sheet name'))
|
||||
#return super(CsvDataSource, self).clean(*args, **kwargs)
|
||||
return "TODO"
|
||||
return super(LDAPResource, self).clean(*args, **kwargs)
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
file_type = self.ldif_file.name.split('.')[-1]
|
||||
#file_type = self.ldif_file.name.split('.')[-1]
|
||||
#if file_type not in ('ods', 'xls', 'xlsx'):
|
||||
# content = self.get_content_without_bom()
|
||||
# dialect = csv.Sniffer().sniff(content)
|
||||
# self.dialect_options = {
|
||||
# k: v for k, v in vars(dialect).items() if not k.startswith('_')
|
||||
# }
|
||||
#return super(CsvDataSource, self).save(*args, **kwargs)
|
||||
return "TODO"
|
||||
return super(LDAPResource, self).save(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def dialect_options(self):
|
||||
def dialect_options(self): #FIXME
|
||||
"""turn dict items into string
|
||||
"""
|
||||
# Set dialect_options if None
|
||||
|
|
@ -102,219 +102,192 @@ class LDAPResource(BaseResource):
|
|||
def dialect_options(self, value):
|
||||
self._dialect_options = value
|
||||
|
||||
def get_content_without_bom(self):
|
||||
def get_content_without_bom(self): # Useless here ?
|
||||
self.ldif_file.seek(0)
|
||||
content = self.ldif_file.read()
|
||||
return content.decode('utf-8-sig', 'ignore').encode('utf-8')
|
||||
|
||||
def get_rows(self):
|
||||
#file_type = self.csv_file.name.split('.')[-1]
|
||||
#if file_type not in ('ods', 'xls', 'xlsx'):
|
||||
# content = self.get_content_without_bom()
|
||||
# reader = csv.reader(content.splitlines(), **self.dialect_options)
|
||||
# rows = list(reader)
|
||||
|
||||
#else:
|
||||
# if file_type == 'ods':
|
||||
# content = get_data_ods(self.csv_file)
|
||||
|
||||
# elif file_type == 'xls' or file_type == 'xlsx':
|
||||
# content = get_data_xls(self.csv_file.path)
|
||||
|
||||
# if self.sheet_name not in content:
|
||||
# return []
|
||||
|
||||
# rows = content[self.sheet_name]
|
||||
|
||||
#if not rows:
|
||||
# return []
|
||||
|
||||
#if self.skip_header:
|
||||
# rows = rows[1:]
|
||||
|
||||
#return [[smart_text(x) for x in y] for y in rows]
|
||||
return "TODO"
|
||||
|
||||
def get_data(self, filters=None):
|
||||
titles = [t.strip() for t in self.columns_keynames.split(',')]
|
||||
indexes = [titles.index(t) for t in titles if t]
|
||||
caption = [titles[i] for i in indexes]
|
||||
#titles = [t.strip() for t in self.columns_keynames.split(',')] #FIXME
|
||||
#indexes = [titles.index(t) for t in titles if t] #FIXME
|
||||
#caption = [titles[i] for i in indexes] #FIXME
|
||||
|
||||
# validate filters (appropriate columns must exist)
|
||||
if filters:
|
||||
for filter_key in filters.keys():
|
||||
if not filter_key.split(lookups.DELIMITER)[0] in titles:
|
||||
del filters[filter_key]
|
||||
## validate filters (appropriate columns must exist)
|
||||
#if filters:
|
||||
# for filter_key in filters.keys():
|
||||
# if not filter_key.split(lookups.DELIMITER)[0] in titles:
|
||||
# del filters[filter_key]
|
||||
|
||||
rows = self.get_rows()
|
||||
#rows = self.get_rows()
|
||||
|
||||
data = []
|
||||
#data = []
|
||||
|
||||
# build a generator of all filters
|
||||
def filters_generator(filters, titles):
|
||||
if not filters:
|
||||
return
|
||||
for key, value in filters.items():
|
||||
try:
|
||||
key, op = key.split(lookups.DELIMITER)
|
||||
except (ValueError,):
|
||||
op = 'eq'
|
||||
## build a generator of all filters
|
||||
#def filters_generator(filters, titles): #FIXME
|
||||
# if not filters:
|
||||
# return
|
||||
# for key, value in filters.items():
|
||||
# try:
|
||||
# key, op = key.split(lookups.DELIMITER)
|
||||
# except (ValueError,):
|
||||
# op = 'eq'
|
||||
|
||||
index = titles.index(key)
|
||||
# index = titles.index(key)
|
||||
|
||||
yield lookups.get_lookup(op, index, value)
|
||||
# yield lookups.get_lookup(op, index, value)
|
||||
|
||||
# apply filters to data
|
||||
def super_filter(filters, data):
|
||||
for f in filters:
|
||||
data = itertools.ifilter(f, data)
|
||||
return data
|
||||
## apply filters to data #FIXME
|
||||
#def super_filter(filters, data):
|
||||
# for f in filters:
|
||||
# data = itertools.ifilter(f, data)
|
||||
# return data
|
||||
|
||||
matches = super_filter(
|
||||
filters_generator(filters, titles), rows
|
||||
)
|
||||
#matches = super_filter(
|
||||
# filters_generator(filters, titles), rows
|
||||
#)
|
||||
|
||||
for row in matches:
|
||||
line = []
|
||||
for i in indexes:
|
||||
try:
|
||||
line.append(row[i])
|
||||
except IndexError:
|
||||
line.append('')
|
||||
#for row in matches:
|
||||
# line = []
|
||||
# for i in indexes:
|
||||
# try:
|
||||
# line.append(row[i])
|
||||
# except IndexError:
|
||||
# line.append('')
|
||||
|
||||
data.append(dict(zip(caption, line)))
|
||||
# data.append(dict(zip(caption, line)))
|
||||
|
||||
return data
|
||||
#return data
|
||||
return 0
|
||||
|
||||
@property
|
||||
def titles(self):
|
||||
return [smart_text(t.strip()) for t in self.columns_keynames.split(',')]
|
||||
#@property
|
||||
#def titles(self): #FIXME
|
||||
# return [smart_text(t.strip()) for t in self.columns_keynames.split(',')]
|
||||
|
||||
@endpoint('json-api', perm='can_access', methods=['get'],
|
||||
name='query', pattern='^(?P<query_name>[\w-]+)/$')
|
||||
def select(self, request, query_name, **kwargs):
|
||||
try:
|
||||
query = Query.objects.get(resource=self.id, slug=query_name)
|
||||
except Query.DoesNotExist:
|
||||
raise APIError(u'no such query')
|
||||
#@endpoint('json-api', perm='can_access', methods=['get'],
|
||||
# name='query', pattern='^(?P<query_name>[\w-]+)/$')
|
||||
#def select(self, request, query_name, **kwargs):
|
||||
# try:
|
||||
# query = Query.objects.get(resource=self.id, slug=query_name)
|
||||
# except Query.DoesNotExist:
|
||||
# raise APIError(u'no such query')
|
||||
|
||||
titles = self.titles
|
||||
rows = self.get_rows()
|
||||
data = [dict(zip(titles, line)) for line in rows]
|
||||
# titles = self.titles
|
||||
# rows = self.get_rows()
|
||||
# data = [dict(zip(titles, line)) for line in rows]
|
||||
|
||||
def stream_expressions(expressions, data, kind, titles=None):
|
||||
codes = []
|
||||
for i, expr in enumerate(expressions):
|
||||
try:
|
||||
code = get_code(expr)
|
||||
except (TypeError, SyntaxError) as e:
|
||||
data = {
|
||||
'expr': expr,
|
||||
'error': unicode(e)
|
||||
}
|
||||
if titles:
|
||||
data['name'] = titles[i]
|
||||
else:
|
||||
data['idx'] = i
|
||||
raise APIError(u'invalid %s expression' % kind, data=data)
|
||||
codes.append((code, expr))
|
||||
for row in data:
|
||||
new_row = []
|
||||
row_vars = dict(row)
|
||||
row_vars['query'] = kwargs
|
||||
for i, (code, expr) in enumerate(codes):
|
||||
try:
|
||||
result = eval(code, {}, row_vars)
|
||||
except Exception as e:
|
||||
data = {
|
||||
'expr': expr,
|
||||
'row': repr(row),
|
||||
}
|
||||
if titles:
|
||||
data['name'] = titles[i]
|
||||
else:
|
||||
data['idx'] = i
|
||||
raise APIError(u'invalid %s expression' % kind, data=data)
|
||||
new_row.append(result)
|
||||
yield new_row, row
|
||||
# def stream_expressions(expressions, data, kind, titles=None):
|
||||
# codes = []
|
||||
# for i, expr in enumerate(expressions):
|
||||
# try:
|
||||
# code = get_code(expr)
|
||||
# except (TypeError, SyntaxError) as e:
|
||||
# data = {
|
||||
# 'expr': expr,
|
||||
# 'error': unicode(e)
|
||||
# }
|
||||
# if titles:
|
||||
# data['name'] = titles[i]
|
||||
# else:
|
||||
# data['idx'] = i
|
||||
# raise APIError(u'invalid %s expression' % kind, data=data)
|
||||
# codes.append((code, expr))
|
||||
# for row in data:
|
||||
# new_row = []
|
||||
# row_vars = dict(row)
|
||||
# row_vars['query'] = kwargs
|
||||
# for i, (code, expr) in enumerate(codes):
|
||||
# try:
|
||||
# result = eval(code, {}, row_vars)
|
||||
# except Exception as e:
|
||||
# data = {
|
||||
# 'expr': expr,
|
||||
# 'row': repr(row),
|
||||
# }
|
||||
# if titles:
|
||||
# data['name'] = titles[i]
|
||||
# else:
|
||||
# data['idx'] = i
|
||||
# raise APIError(u'invalid %s expression' % kind, data=data)
|
||||
# new_row.append(result)
|
||||
# yield new_row, row
|
||||
|
||||
filters = query.get_list('filters')
|
||||
# filters = query.get_list('filters')
|
||||
|
||||
if filters:
|
||||
data = [row for new_row, row in stream_expressions(filters, data, kind='filters')
|
||||
if all(new_row)]
|
||||
# if filters:
|
||||
# data = [row for new_row, row in stream_expressions(filters, data, kind='filters')
|
||||
# if all(new_row)]
|
||||
|
||||
order = query.get_list('order')
|
||||
if order:
|
||||
generator = stream_expressions(order, data, kind='order')
|
||||
new_data = [(tuple(new_row), row) for new_row, row in generator]
|
||||
new_data.sort(key=lambda (k, row): k)
|
||||
data = [row for key, row in new_data]
|
||||
# order = query.get_list('order')
|
||||
# if order:
|
||||
# generator = stream_expressions(order, data, kind='order')
|
||||
# new_data = [(tuple(new_row), row) for new_row, row in generator]
|
||||
# new_data.sort(key=lambda (k, row): k)
|
||||
# data = [row for key, row in new_data]
|
||||
|
||||
distinct = query.get_list('distinct')
|
||||
if distinct:
|
||||
generator = stream_expressions(distinct, data, kind='distinct')
|
||||
seen = set()
|
||||
new_data = []
|
||||
for new_row, row in generator:
|
||||
new_row = tuple(new_row)
|
||||
try:
|
||||
hash(new_row)
|
||||
except TypeError:
|
||||
raise APIError(u'distinct value is unhashable',
|
||||
data={
|
||||
'row': repr(row),
|
||||
'distinct': repr(new_row),
|
||||
})
|
||||
if new_row in seen:
|
||||
continue
|
||||
new_data.append(row)
|
||||
seen.add(new_row)
|
||||
data = new_data
|
||||
# distinct = query.get_list('distinct')
|
||||
# if distinct:
|
||||
# generator = stream_expressions(distinct, data, kind='distinct')
|
||||
# seen = set()
|
||||
# new_data = []
|
||||
# for new_row, row in generator:
|
||||
# new_row = tuple(new_row)
|
||||
# try:
|
||||
# hash(new_row)
|
||||
# except TypeError:
|
||||
# raise APIError(u'distinct value is unhashable',
|
||||
# data={
|
||||
# 'row': repr(row),
|
||||
# 'distinct': repr(new_row),
|
||||
# })
|
||||
# if new_row in seen:
|
||||
# continue
|
||||
# new_data.append(row)
|
||||
# seen.add(new_row)
|
||||
# data = new_data
|
||||
|
||||
projection = query.get_list('projections')
|
||||
if projection:
|
||||
expressions = []
|
||||
titles = []
|
||||
for mapping in projection:
|
||||
name, expr = mapping.split(':', 1)
|
||||
if not identifier_re.match(name):
|
||||
raise APIError(u'invalid projection name', data=name)
|
||||
titles.append(name)
|
||||
expressions.append(expr)
|
||||
new_data = []
|
||||
for new_row, row in stream_expressions(expressions, data, kind='projection',
|
||||
titles=titles):
|
||||
new_data.append(dict(zip(titles, new_row)))
|
||||
data = new_data
|
||||
# projection = query.get_list('projections')
|
||||
# if projection:
|
||||
# expressions = []
|
||||
# titles = []
|
||||
# for mapping in projection:
|
||||
# name, expr = mapping.split(':', 1)
|
||||
# if not identifier_re.match(name):
|
||||
# raise APIError(u'invalid projection name', data=name)
|
||||
# titles.append(name)
|
||||
# expressions.append(expr)
|
||||
# new_data = []
|
||||
# for new_row, row in stream_expressions(expressions, data, kind='projection',
|
||||
# titles=titles):
|
||||
# new_data.append(dict(zip(titles, new_row)))
|
||||
# data = new_data
|
||||
|
||||
# allow jsonp queries by select2
|
||||
# filtering is done there afater projection because we need a projection named text for
|
||||
# retro-compatibility with previous use of the csvdatasource with select2
|
||||
if 'q' in request.GET:
|
||||
if 'case-insensitive' in request.GET:
|
||||
filters = ["query['q'].lower() in text.lower()"]
|
||||
else:
|
||||
filters = ["query['q'] in text"]
|
||||
data = [row for new_row, row in stream_expressions(filters, data, kind='filters')
|
||||
if new_row[0]]
|
||||
# # allow jsonp queries by select2
|
||||
# # filtering is done there afater projection because we need a projection named text for
|
||||
# # retro-compatibility with previous use of the csvdatasource with select2
|
||||
# if 'q' in request.GET:
|
||||
# if 'case-insensitive' in request.GET:
|
||||
# filters = ["query['q'].lower() in text.lower()"]
|
||||
# else:
|
||||
# filters = ["query['q'] in text"]
|
||||
# data = [row for new_row, row in stream_expressions(filters, data, kind='filters')
|
||||
# if new_row[0]]
|
||||
|
||||
if query.structure == 'array':
|
||||
return [[row[t] for t in titles] for row in data]
|
||||
elif query.structure == 'dict':
|
||||
return data
|
||||
elif query.structure == 'tuples':
|
||||
return [[[t, row[t]] for t in titles] for row in data]
|
||||
elif query.structure == 'onerow':
|
||||
if len(data) != 1:
|
||||
raise APIError('more or less than one row', data=data)
|
||||
return data[0]
|
||||
elif query.structure == 'one':
|
||||
if len(data) != 1:
|
||||
raise APIError('more or less than one row', data=data)
|
||||
if len(data[0]) != 1:
|
||||
raise APIError('more or less than one column', data=data)
|
||||
return data[0].values()[0]
|
||||
# if query.structure == 'array':
|
||||
# return [[row[t] for t in titles] for row in data]
|
||||
# elif query.structure == 'dict':
|
||||
# return data
|
||||
# elif query.structure == 'tuples':
|
||||
# return [[[t, row[t]] for t in titles] for row in data]
|
||||
# elif query.structure == 'onerow':
|
||||
# if len(data) != 1:
|
||||
# raise APIError('more or less than one row', data=data)
|
||||
# return data[0]
|
||||
# elif query.structure == 'one':
|
||||
# if len(data) != 1:
|
||||
# raise APIError('more or less than one row', data=data)
|
||||
# if len(data[0]) != 1:
|
||||
# raise APIError('more or less than one column', data=data)
|
||||
# return data[0].values()[0]
|
||||
|
||||
class Query(models.Model):
|
||||
resource = models.ForeignKey('LDAPResource')
|
||||
|
|
|
|||
Reference in New Issue