entrouvert
/
bijoe
17
0
Fork 0
Code Issues Pull Requests Releases Activity

misc: apply pyupgrade/isort/black (#56062)

This commit is contained in:
Benjamin Dauvergne 2021-08-09 15:31:51 +02:00
parent 951bd3387f
commit 65d33f00b7
41 changed files with 922 additions and 787 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
bijoe/engine.py
View File

@ -17,14 +17,13 @@
import collections import collections
import contextlib import contextlib
import datetime import datetime
import logging
import itertools
import hashlib import hashlib
import itertools
import logging
import psycopg2 import psycopg2
from django.core.cache import cache
from django.conf import settings from django.conf import settings
from django.core.cache import cache
from django.utils.encoding import force_bytes, force_text from django.utils.encoding import force_bytes, force_text
from django.utils.six import python_2_unicode_compatible from django.utils.six import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
@ -63,20 +62,20 @@ class MeasureCell(collections.namedtuple('_Cell', ['measure', 'value'])):
return _('N/A') return _('N/A')
else: else:
try: try:
return (u'%4.2f' % float(value)).replace('.', ',') + u' %' return ('%4.2f' % float(value)).replace('.', ',') + ' %'
except TypeError: except TypeError:
return _('N/A') return _('N/A')
elif self.measure.type == 'duration': elif self.measure.type == 'duration':
if value is None: if value is None:
return u'0' return '0'
else: else:
s = u'' s = ''
if value.days: if value.days:
s += u'%d jour(s)' % value.days s += '%d jour(s)' % value.days
if value.seconds // 3600: if value.seconds // 3600:
s += u' %d heure(s)' % (value.seconds // 3600) s += ' %d heure(s)' % (value.seconds // 3600)
if not s: if not s:
s = u'moins d\'1 heure' s = 'moins d\'1 heure'
return s return s
elif self.measure.type == 'bool': elif self.measure.type == 'bool':
if value is None: if value is None:
@ -99,7 +98,7 @@ class Cells(collections.namedtuple('Cells', ['dimensions', 'measures'])):
def __new__(cls, dimensions=[], measures=[]): def __new__(cls, dimensions=[], measures=[]):
dimensions = list(dimensions) dimensions = list(dimensions)
measures = list(measures) measures = list(measures)
return super(Cells, cls).__new__(cls, dimensions, measures) return super().__new__(cls, dimensions, measures)
def quote(s): def quote(s):
@ -126,7 +125,7 @@ def to_tuple(cur, values):
Member = collections.namedtuple('Member', ['id', 'label']) Member = collections.namedtuple('Member', ['id', 'label'])
class EngineDimension(object): class EngineDimension:
def __init__(self, engine, engine_cube, dimension): def __init__(self, engine, engine_cube, dimension):
self.engine = engine self.engine = engine
self.engine_cube = engine_cube self.engine_cube = engine_cube
@ -148,9 +147,13 @@ class EngineDimension(object):
cache_key = self.cache_key(filters) cache_key = self.cache_key(filters)
members = cache.get(cache_key) members = cache.get(cache_key)
if members is not None: if members is not None:
self.engine.log.debug('MEMBERS: (from cache) dimension %s.%s filters=%s: %s', self.engine.log.debug(
self.engine_cube.name, self.name, filters, 'MEMBERS: (from cache) dimension %s.%s filters=%s: %s',
members) self.engine_cube.name,
self.name,
filters,
members,
)
return members return members
members = [] members = []
@ -184,7 +187,8 @@ class EngineDimension(object):
table_expression = '%s' % self.engine_cube.fact_table table_expression = '%s' % self.engine_cube.fact_table
if joins: if joins:
table_expression = self.engine_cube.build_table_expression( table_expression = self.engine_cube.build_table_expression(
joins, self.engine_cube.fact_table, force_join='right') joins, self.engine_cube.fact_table, force_join='right'
)
sql = 'SELECT %s AS value, %s::text AS label ' % (value, value_label) sql = 'SELECT %s AS value, %s::text AS label ' % (value, value_label)
sql += 'FROM %s ' % table_expression sql += 'FROM %s ' % table_expression
if order_by: if order_by:
@ -199,7 +203,7 @@ class EngineDimension(object):
if order_value not in group_by: if order_value not in group_by:
group_by.append(order_value) group_by.append(order_value)
if conditions: if conditions:
sql += 'WHERE %s ' % (' AND '.join(conditions)) sql += 'WHERE %s ' % ' AND '.join(conditions)
sql += 'GROUP BY %s ' % ', '.join(group_by) sql += 'GROUP BY %s ' % ', '.join(group_by)
sql += 'ORDER BY (%s) ' % ', '.join(order_by) sql += 'ORDER BY (%s) ' % ', '.join(order_by)
sql = sql.format(fact_table=self.engine_cube.fact_table) sql = sql.format(fact_table=self.engine_cube.fact_table)
@ -210,14 +214,15 @@ class EngineDimension(object):
continue continue
members.append(Member(id=row[0], label=force_text(row[1]))) members.append(Member(id=row[0], label=force_text(row[1])))
cache.set(cache_key, members, 600) cache.set(cache_key, members, 600)
self.engine.log.debug('MEMBERS: dimension %s.%s filters=%s: %s', self.engine.log.debug(
self.engine_cube.name, self.name, filters, 'MEMBERS: dimension %s.%s filters=%s: %s', self.engine_cube.name, self.name, filters, members
members) )
return members return members
class SchemaJSONDimension(schemas.Dimension): class SchemaJSONDimension(schemas.Dimension):
'''Generated dimensions for JSON fields keys''' '''Generated dimensions for JSON fields keys'''
filter = False filter = False
order_by = None order_by = None
group_by = None group_by = None
@ -225,7 +230,7 @@ class SchemaJSONDimension(schemas.Dimension):
type = 'string' type = 'string'
def __init__(self, json_field, name): def __init__(self, json_field, name):
super(SchemaJSONDimension, self).__init__() super().__init__()
name = str(name) name = str(name)
self.name = name self.name = name
self.label = name.title() self.label = name.title()
@ -233,27 +238,30 @@ class SchemaJSONDimension(schemas.Dimension):
self.value_label = expr self.value_label = expr
self.value = expr self.value = expr
self.join = ['json_' + name] self.join = ['json_' + name]
sql = ('SELECT DISTINCT {json_field}->>\'%s\' AS v, {json_field}->>\'%s\' AS v' sql = (
' FROM {{fact_table}} WHERE ({json_field}->>\'%s\') IS NOT NULL ORDER BY v' % 'SELECT DISTINCT {json_field}->>\'%s\' AS v, {json_field}->>\'%s\' AS v'
(self.name, self.name, self.name)) ' FROM {{fact_table}} WHERE ({json_field}->>\'%s\') IS NOT NULL ORDER BY v'
% (self.name, self.name, self.name)
)
self.members_query = sql.format(json_field=json_field) self.members_query = sql.format(json_field=json_field)
self.filter_expression = ('({fact_table}.id IS NULL ' self.filter_expression = '({fact_table}.id IS NULL OR ({fact_table}.%s->>\'%s\') IN (%%s))' % (
'OR ({fact_table}.%s->>\'%s\') IN (%%s))' json_field,
% (json_field, name)) name,
)
self.filter_needs_join = False self.filter_needs_join = False
self.absent_label = _('None') self.absent_label = _('None')
class EngineJSONDimension(EngineDimension): class EngineJSONDimension(EngineDimension):
def __init__(self, engine, engine_cube, name): def __init__(self, engine, engine_cube, name):
self.engine = engine self.engine = engine
self.engine_cube = engine_cube self.engine_cube = engine_cube
self.dimension = SchemaJSONDimension(self.engine_cube.json_field, name) self.dimension = SchemaJSONDimension(self.engine_cube.json_field, name)
def cache_key(self, filters): def cache_key(self, filters):
key = (self.engine.path + self.engine_cube.json_field key = (
+ self.engine_cube.name + self.name + repr(filters)) self.engine.path + self.engine_cube.json_field + self.engine_cube.name + self.name + repr(filters)
)
return hashlib.md5(force_bytes(key)).hexdigest() return hashlib.md5(force_bytes(key)).hexdigest()
def to_json(self): def to_json(self):
@ -263,7 +271,7 @@ class EngineJSONDimension(EngineDimension):
} }
class EngineMeasure(object): class EngineMeasure:
def __init__(self, engine, engine_cube, measure): def __init__(self, engine, engine_cube, measure):
self.engine = engine self.engine = engine
self.engine_cube = engine_cube self.engine_cube = engine_cube
@ -273,7 +281,7 @@ class EngineMeasure(object):
return getattr(self.measure, name) return getattr(self.measure, name)
class ProxyList(object): class ProxyList:
chain = None chain = None
def __init__(self, engine, engine_cube, attribute, cls, chain=None): def __init__(self, engine, engine_cube, attribute, cls, chain=None):
@ -285,8 +293,9 @@ class ProxyList(object):
self.chain = chain(engine, engine_cube) self.chain = chain(engine, engine_cube)
def __iter__(self): def __iter__(self):
i = (self.cls(self.engine, self.engine_cube, o) i = (
for o in getattr(self.engine_cube.cube, self.attribute)) self.cls(self.engine, self.engine_cube, o) for o in getattr(self.engine_cube.cube, self.attribute)
)
if self.chain: if self.chain:
i = itertools.chain(i, self.chain) i = itertools.chain(i, self.chain)
return i return i
@ -300,7 +309,7 @@ class ProxyList(object):
raise KeyError raise KeyError
class JSONDimensions(object): class JSONDimensions:
__cache = None __cache = None
def __init__(self, engine, engine_cube): def __init__(self, engine, engine_cube):
@ -313,8 +322,10 @@ class JSONDimensions(object):
return [] return []
if not self.__cache: if not self.__cache:
with self.engine.get_cursor() as cursor: with self.engine.get_cursor() as cursor:
sql = ('select distinct jsonb_object_keys(%s) as a from %s order by a' sql = 'select distinct jsonb_object_keys(%s) as a from %s order by a' % (
% (self.engine_cube.json_field, self.engine_cube.fact_table)) self.engine_cube.json_field,
self.engine_cube.fact_table,
)
cursor.execute(sql) cursor.execute(sql)
self.__cache = [row[0] for row in cursor.fetchall()] self.__cache = [row[0] for row in cursor.fetchall()]
return self.__cache return self.__cache
@ -329,7 +340,7 @@ class JSONDimensions(object):
return EngineJSONDimension(self.engine, self.engine_cube, name) return EngineJSONDimension(self.engine, self.engine_cube, name)
class ProxyListDescriptor(object): class ProxyListDescriptor:
def __init__(self, attribute, cls, chain=None): def __init__(self, attribute, cls, chain=None):
self.attribute = attribute self.attribute = attribute
self.cls = cls self.cls = cls
@ -342,7 +353,7 @@ class ProxyListDescriptor(object):
return obj.__dict__[key] return obj.__dict__[key]
class EngineCube(object): class EngineCube:
dimensions = ProxyListDescriptor('all_dimensions', EngineDimension, chain=JSONDimensions) dimensions = ProxyListDescriptor('all_dimensions', EngineDimension, chain=JSONDimensions)
measures = ProxyListDescriptor('measures', EngineMeasure) measures = ProxyListDescriptor('measures', EngineMeasure)
@ -365,10 +376,16 @@ class EngineCube(object):
name=name, name=name,
table=( table=(
'(SELECT DISTINCT %s.%s->>\'%s\' AS value FROM %s ' '(SELECT DISTINCT %s.%s->>\'%s\' AS value FROM %s '
'WHERE (%s.%s->>\'%s\') IS NOT NULL ORDER BY value)' % ( 'WHERE (%s.%s->>\'%s\') IS NOT NULL ORDER BY value)'
self.fact_table, self.json_field, json_key, self.fact_table, )
self.fact_table, self.json_field, json_key % (
) self.fact_table,
self.json_field,
json_key,
self.fact_table,
self.fact_table,
self.json_field,
json_key,
), ),
master='%s->>\'%s\'' % (self.json_field, json_key), master='%s->>\'%s\'' % (self.json_field, json_key),
detail='value', detail='value',
@ -427,15 +444,22 @@ class EngineCube(object):
sql += ' GROUP BY %s' % ', '.join(group_by) sql += ' GROUP BY %s' % ', '.join(group_by)
if order_by: if order_by:
sql += ' ORDER BY %s' % ', '.join(order_by) sql += ' ORDER BY %s' % ', '.join(order_by)
sql = sql.format(fact_table=self.cube.fact_table, sql = sql.format(
table_expression=table_expression, fact_table=self.cube.fact_table,
where_conditions=where_conditions) table_expression=table_expression,
where_conditions=where_conditions,
)
return sql return sql
def query(self, filters, drilldown, measures, **kwargs): def query(self, filters, drilldown, measures, **kwargs):
self.engine.log.debug('%s.%s query filters=%s drilldown=%s measures=%s', self.engine.log.debug(
self.engine.warehouse.name, self.cube.name, filters, drilldown, '%s.%s query filters=%s drilldown=%s measures=%s',
measures) self.engine.warehouse.name,
self.cube.name,
filters,
drilldown,
measures,
)
with self.engine.get_cursor() as cursor: with self.engine.get_cursor() as cursor:
sql = self.sql_query(filters=filters, drilldown=drilldown, measures=measures, **kwargs) sql = self.sql_query(filters=filters, drilldown=drilldown, measures=measures, **kwargs)
self.engine.log.debug('SQL: %s', sql) self.engine.log.debug('SQL: %s', sql)
@ -451,16 +475,20 @@ class EngineCube(object):
else: else:
value_label = row[j + 1] value_label = row[j + 1]
j += 2 j += 2
cells.dimensions.append(DimensionCell( cells.dimensions.append(
dimension=dimension, DimensionCell(
value=value, dimension=dimension,
value_label=value_label, value=value,
)) value_label=value_label,
)
)
for i, measure in enumerate(measures): for i, measure in enumerate(measures):
cells.measures.append(MeasureCell( cells.measures.append(
measure=measure, MeasureCell(
value=row[j + i], measure=measure,
)) value=row[j + i],
)
)
yield cells yield cells
JOIN_KINDS = { JOIN_KINDS = {
@ -471,8 +499,8 @@ class EngineCube(object):
} }
def build_table_expression(self, joins, table_name, force_join=None): def build_table_expression(self, joins, table_name, force_join=None):
'''Recursively build the table expression from the join tree, """Recursively build the table expression from the join tree,
starting from the fact table''' starting from the fact table"""
join_tree = {} join_tree = {}
# Build join tree # Build join tree
@ -495,12 +523,15 @@ class EngineCube(object):
for join_name, join in joins.items(): for join_name, join in joins.items():
contain_joins = True contain_joins = True
sql += ' %s ' % self.JOIN_KINDS[kind] sql += ' %s ' % self.JOIN_KINDS[kind]
sql += ' ' + build_table_expression_helper(join_tree, join.table, alias=join.name, top=False) sql += ' ' + build_table_expression_helper(
join_tree, join.table, alias=join.name, top=False
)
condition = '%s.%s = %s.%s' % ( condition = '%s.%s = %s.%s' % (
alias or table_name, alias or table_name,
join.master.split('.')[-1], join.master.split('.')[-1],
quote(join.name), quote(join.name),
join.detail) join.detail,
)
sql += ' ON ' + condition sql += ' ON ' + condition
# if the table expression contains joins and is not the full table # if the table expression contains joins and is not the full table
@ -514,7 +545,7 @@ class EngineCube(object):
return build_table_expression_helper(join_tree, table_name) return build_table_expression_helper(join_tree, table_name)
class Engine(object): class Engine:
def __init__(self, warehouse): def __init__(self, warehouse):
self.warehouse = warehouse self.warehouse = warehouse
self.log = logging.getLogger(__name__) self.log = logging.getLogger(__name__)

41
bijoe/hobo_agent/management/commands/hobo_deploy.py
View File

@ -25,12 +25,11 @@ try:
except ImportError: except ImportError:
sentry_sdk = None sentry_sdk = None
from tenant_schemas.utils import tenant_context from django.conf import settings
from django.utils.encoding import force_str
from hobo.agent.common.management.commands import hobo_deploy from hobo.agent.common.management.commands import hobo_deploy
from hobo.multitenant.settings_loaders import KnownServices from hobo.multitenant.settings_loaders import KnownServices
from tenant_schemas.utils import tenant_context
from django.utils.encoding import force_str
from django.conf import settings
def pg_dsn_quote(value): def pg_dsn_quote(value):
@ -47,9 +46,10 @@ def truncate_pg_identifier(identifier, hash_length=6, force_hash=False):
else: else:
# insert hash in the middle, to keep some readability # insert hash in the middle, to keep some readability
return ( return (
identifier[:(63 - hash_length) // 2] identifier[: (63 - hash_length) // 2]
+ hashlib.md5(identifier.encode('utf-8')).hexdigest()[:hash_length] + hashlib.md5(identifier.encode('utf-8')).hexdigest()[:hash_length]
+ identifier[-(63 - hash_length) // 2:]) + identifier[-(63 - hash_length) // 2 :]
)
def schema_from_url(url, hash_length=6): def schema_from_url(url, hash_length=6):
@ -62,7 +62,7 @@ def schema_from_url(url, hash_length=6):
class Command(hobo_deploy.Command): class Command(hobo_deploy.Command):
def deploy_specifics(self, hobo_environment, tenant): def deploy_specifics(self, hobo_environment, tenant):
super(Command, self).deploy_specifics(hobo_environment, tenant) super().deploy_specifics(hobo_environment, tenant)
with tenant_context(tenant): with tenant_context(tenant):
services = hobo_environment.get('services') services = hobo_environment.get('services')
ini_file = os.path.join(tenant.get_directory(), 'wcs-olap.ini') ini_file = os.path.join(tenant.get_directory(), 'wcs-olap.ini')
@ -77,15 +77,18 @@ class Command(hobo_deploy.Command):
config.add_section('wcs-olap') config.add_section('wcs-olap')
config.set('wcs-olap', 'cubes_model_dirs', schemas_path) config.set('wcs-olap', 'cubes_model_dirs', schemas_path)
pg_dsn_parts = [] pg_dsn_parts = []
for pg_dsn_part in [('NAME', 'dbname'), for pg_dsn_part in [
('HOST', 'host'), ('NAME', 'dbname'),
('USER', 'user'), ('HOST', 'host'),
('PASSWORD', 'password'), ('USER', 'user'),
('PORT', 'port')]: ('PASSWORD', 'password'),
('PORT', 'port'),
]:
if settings.DATABASES['default'].get(pg_dsn_part[0]): if settings.DATABASES['default'].get(pg_dsn_part[0]):
pg_dsn_parts.append('%s=%s' % ( pg_dsn_parts.append(
pg_dsn_part[1], '%s=%s'
pg_dsn_quote(settings.DATABASES['default'].get(pg_dsn_part[0])))) % (pg_dsn_part[1], pg_dsn_quote(settings.DATABASES['default'].get(pg_dsn_part[0])))
)
config.set('wcs-olap', 'pg_dsn', config_parser_quote(' '.join(pg_dsn_parts))) config.set('wcs-olap', 'pg_dsn', config_parser_quote(' '.join(pg_dsn_parts)))
for service in services: for service in services:
@ -97,8 +100,12 @@ class Command(hobo_deploy.Command):
our_key = this['secret_key'] our_key = this['secret_key']
for service in services: for service in services:
base_url = service.get('base_url') base_url = service.get('base_url')
if (service.get('this') or not service.get('secret_key') if (
or service['service-id'] != 'wcs' or not service.get('base_url')): service.get('this')
or not service.get('secret_key')
or service['service-id'] != 'wcs'
or not service.get('base_url')
):
continue continue
elif service.get('secondary') and not config.has_section(base_url): elif service.get('secondary') and not config.has_section(base_url):
# skip secondary instances unless they were already added, # skip secondary instances unless they were already added,

4
bijoe/management/commands/export_site.py
View File

@ -27,8 +27,8 @@ class Command(BaseCommand):
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument( parser.add_argument(
'--output', metavar='FILE', default=None, '--output', metavar='FILE', default=None, help='name of a file to write output to'
help='name of a file to write output to') )
def handle(self, *args, **options): def handle(self, *args, **options):
if options['output']: if options['output']:

12
bijoe/management/commands/import_site.py
View File

@ -26,15 +26,11 @@ class Command(BaseCommand):
help = 'Import an exported site' help = 'Import an exported site'
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('filename', metavar='FILENAME', type=str, help='name of file to import')
parser.add_argument('--clean', action='store_true', default=False, help='Clean site before importing')
parser.add_argument( parser.add_argument(
'filename', metavar='FILENAME', type=str, '--if-empty', action='store_true', default=False, help='Import only if site is empty'
help='name of file to import') )
parser.add_argument(
'--clean', action='store_true', default=False,
help='Clean site before importing')
parser.add_argument(
'--if-empty', action='store_true', default=False,
help='Import only if site is empty')
def handle(self, filename, **options): def handle(self, filename, **options):
if filename == '-': if filename == '-':

12
bijoe/relative_time.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# bijoe - BI dashboard # bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert # Copyright (C) 2015 Entr'ouvert
# #
@ -17,23 +16,24 @@
import re import re
from datetime import date from datetime import date
from dateutil.relativedelta import relativedelta, MO
import isodate import isodate
from dateutil.relativedelta import MO, relativedelta
class RelativeDate(date): class RelativeDate(date):
__TEMPLATES = [ __TEMPLATES = [
{ {
'pattern': u' *cette +année *', 'pattern': ' *cette +année *',
'truncate': 'year', 'truncate': 'year',
}, },
{ {
'pattern': u' *l\'année +dernière *', 'pattern': ' *l\'année +dernière *',
'truncate': 'year', 'truncate': 'year',
'timedelta': {'years': -1}, 'timedelta': {'years': -1},
}, },
{ {
'pattern': u' *l\'année +prochaine *', 'pattern': ' *l\'année +prochaine *',
'truncate': 'year', 'truncate': 'year',
'timedelta': {'years': 1}, 'timedelta': {'years': 1},
}, },
@ -131,7 +131,7 @@ class RelativeDate(date):
elif group.startswith('-'): elif group.startswith('-'):
sign = -1 sign = -1
group = group[1:] group = group[1:]
n = re.match('(\d+)\*(\w+)', group) n = re.match(r'(\d+)\*(\w+)', group)
try: try:
if n: if n:
value = int(n.group(1) * m.group(n.group(2))) value = int(n.group(1) * m.group(n.group(2)))

136
bijoe/schemas.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# #
# bijoe - BI dashboard # bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert # Copyright (C) 2015 Entr'ouvert
@ -16,9 +15,9 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import collections
import datetime import datetime
import decimal import decimal
import collections
from django.utils import six from django.utils import six
from django.utils.encoding import force_text from django.utils.encoding import force_text
@ -49,7 +48,7 @@ def type_cast(t):
return t return t
class Base(object): class Base:
__types__ = {} __types__ = {}
def __init__(self, **kwargs): def __init__(self, **kwargs):
@ -78,14 +77,16 @@ class Base(object):
def from_json(cls, d): def from_json(cls, d):
assert hasattr(d, 'keys') assert hasattr(d, 'keys')
slots = cls.slots() slots = cls.slots()
assert set(d.keys()) <= set(slots), \ assert set(d.keys()) <= set(slots), 'given keys %r does not match %s.__slots__: %r' % (
'given keys %r does not match %s.__slots__: %r' % (d.keys(), cls.__name__, slots) d.keys(),
cls.__name__,
slots,
)
types = cls.types() types = cls.types()
kwargs = {} kwargs = {}
for key in slots: for key in slots:
assert key in d or hasattr(cls, key), \ assert key in d or hasattr(cls, key), '%s.%s is is a mandatory attribute' % (cls.__name__, key)
'%s.%s is is a mandatory attribute' % (cls.__name__, key)
if not key in d: if not key in d:
continue continue
value = d[key] value = d[key]
@ -133,7 +134,7 @@ class Measure(Base):
__slots__ = ['name', 'label', 'type', 'expression'] __slots__ = ['name', 'label', 'type', 'expression']
__types__ = { __types__ = {
'name': str, 'name': str,
'label': six.text_type, 'label': str,
'type': type_cast, 'type': type_cast,
'expression': str, 'expression': str,
} }
@ -146,12 +147,25 @@ class Measure(Base):
class Dimension(Base): class Dimension(Base):
__slots__ = ['name', 'label', 'type', 'join', 'value', 'value_label', __slots__ = [
'order_by', 'group_by', 'filter_in_join', 'filter', 'filter_value', 'name',
'filter_needs_join', 'filter_expression', 'absent_label'] 'label',
'type',
'join',
'value',
'value_label',
'order_by',
'group_by',
'filter_in_join',
'filter',
'filter_value',
'filter_needs_join',
'filter_expression',
'absent_label',
]
__types__ = { __types__ = {
'name': str, 'name': str,
'label': six.text_type, 'label': str,
'type': str, 'type': str,
'join': [str], 'join': [str],
'value': str, 'value': str,
@ -164,7 +178,7 @@ class Dimension(Base):
'filter_in_join': bool, 'filter_in_join': bool,
'filter_value': str, 'filter_value': str,
'filter_needs_join': bool, 'filter_needs_join': bool,
'absent_label': six.text_type, 'absent_label': str,
} }
def __init__(self, **kwargs): def __init__(self, **kwargs):
@ -180,7 +194,7 @@ class Dimension(Base):
self.filter_needs_join = True self.filter_needs_join = True
self.members_query = None self.members_query = None
self.absent_label = None self.absent_label = None
super(Dimension, self).__init__(**kwargs) super().__init__(**kwargs)
if not self.absent_label: if not self.absent_label:
if self.type in ('date', 'integer', 'string'): if self.type in ('date', 'integer', 'string'):
self.absent_label = _('None') self.absent_label = _('None')
@ -196,33 +210,31 @@ class Dimension(Base):
return [ return [
self, self,
Dimension( Dimension(
label=u'année (%s)' % self.label, label='année (%s)' % self.label,
name=self.name + '__year', name=self.name + '__year',
type='integer', type='integer',
join=self.join, join=self.join,
filter_value='EXTRACT(year from %s)::integer' % filter_value, filter_value='EXTRACT(year from %s)::integer' % filter_value,
filter_in_join=self.filter_in_join, filter_in_join=self.filter_in_join,
value='EXTRACT(year from %s)::integer' % self.value, value='EXTRACT(year from %s)::integer' % self.value,
filter=False), filter=False,
),
Dimension( Dimension(
label=u'année et mois (%s)' % self.label, label='année et mois (%s)' % self.label,
name=self.name + '__yearmonth', name=self.name + '__yearmonth',
type='integer', type='integer',
join=self.join, join=self.join,
filter_value='EXTRACT(year from %s) || \'M\' || EXTRACT(month from %s)' filter_value='EXTRACT(year from %s) || \'M\' || EXTRACT(month from %s)'
% (filter_value, filter_value), % (filter_value, filter_value),
filter_in_join=self.filter_in_join, filter_in_join=self.filter_in_join,
value='TO_CHAR(EXTRACT(month from %s), \'00\') || \'/\' || EXTRACT(year from %s)' value='TO_CHAR(EXTRACT(month from %s), \'00\') || \'/\' || EXTRACT(year from %s)'
% (self.value, self.value), % (self.value, self.value),
group_by='EXTRACT(year from %s), EXTRACT(month from %s)' % (self.value, group_by='EXTRACT(year from %s), EXTRACT(month from %s)' % (self.value, self.value),
self.value), order_by=['EXTRACT(year from %s), EXTRACT(month from %s)' % (self.value, self.value)],
order_by=['EXTRACT(year from %s), EXTRACT(month from %s)' % (self.value, filter=False,
self.value)], ),
filter=False),
Dimension( Dimension(
label=u'mois (%s)' % self.label, label='mois (%s)' % self.label,
name=self.name + '__month', name=self.name + '__month',
type='integer', type='integer',
filter_value='EXTRACT(month from %s)' % filter_value, filter_value='EXTRACT(month from %s)' % filter_value,
@ -230,12 +242,12 @@ class Dimension(Base):
join=self.join, join=self.join,
value='EXTRACT(month from %s)' % self.value, value='EXTRACT(month from %s)' % self.value,
value_label='to_char(date_trunc(\'month\', %s), \'TMmonth\')' % self.value, value_label='to_char(date_trunc(\'month\', %s), \'TMmonth\')' % self.value,
group_by='EXTRACT(month from %s), ' group_by='EXTRACT(month from %s), to_char(date_trunc(\'month\', %s), \'TMmonth\')'
'to_char(date_trunc(\'month\', %s), \'TMmonth\')' % (self.value, self.value),
% (self.value, self.value), filter=False,
filter=False), ),
Dimension( Dimension(
label=u'jour de la semaine (%s)' % self.label, label='jour de la semaine (%s)' % self.label,
name=self.name + '__dow', name=self.name + '__dow',
type='integer', type='integer',
join=self.join, join=self.join,
@ -243,24 +255,27 @@ class Dimension(Base):
filter_in_join=self.filter_in_join, filter_in_join=self.filter_in_join,
value='EXTRACT(dow from %s)' % self.value, value='EXTRACT(dow from %s)' % self.value,
order_by=['(EXTRACT(dow from %s) + 6)::integer %% 7' % self.value], order_by=['(EXTRACT(dow from %s) + 6)::integer %% 7' % self.value],
value_label='to_char(date_trunc(\'week\', current_date)::date ' value_label=(
'+ EXTRACT(dow from %s)::integer - 1, \'TMday\')' % self.value, 'to_char(date_trunc(\'week\', current_date)::date '
filter=False), '+ EXTRACT(dow from %s)::integer - 1, \'TMday\')'
)
% self.value,
filter=False,
),
Dimension( Dimension(
label=u'semaine (%s)' % self.label, label='semaine (%s)' % self.label,
name=self.name + '__isoweek', name=self.name + '__isoweek',
type='integer', type='integer',
join=self.join, join=self.join,
filter_value='EXTRACT(isoyear from %s) || \'S\' || EXTRACT(week from %s)' filter_value='EXTRACT(isoyear from %s) || \'S\' || EXTRACT(week from %s)'
% (filter_value, filter_value), % (filter_value, filter_value),
filter_in_join=self.filter_in_join, filter_in_join=self.filter_in_join,
value='EXTRACT(isoyear from %s) || \'S\' || EXTRACT(week from %s)' value='EXTRACT(isoyear from %s) || \'S\' || EXTRACT(week from %s)'
% (self.value, self.value), % (self.value, self.value),
group_by='EXTRACT(isoyear from %s), EXTRACT(week from %s)' % (self.value, group_by='EXTRACT(isoyear from %s), EXTRACT(week from %s)' % (self.value, self.value),
self.value), order_by=['EXTRACT(isoyear from %s), EXTRACT(week from %s)' % (self.value, self.value)],
order_by=['EXTRACT(isoyear from %s), EXTRACT(week from %s)' % (self.value, filter=False,
self.value)], ),
filter=False)
] ]
return [self] return [self]
@ -268,8 +283,7 @@ class Dimension(Base):
value = self.filter_value or self.value value = self.filter_value or self.value
if self.type == 'date': if self.type == 'date':
assert isinstance(filter_values, dict) and set(filter_values.keys()) == set(['start', assert isinstance(filter_values, dict) and set(filter_values.keys()) == {'start', 'end'}
'end'])
filters = [] filters = []
values = [] values = []
@ -278,6 +292,7 @@ class Dimension(Base):
filter_value = RelativeDate(filter_value) filter_value = RelativeDate(filter_value)
filters.append(tpl % (value, '%s')) filters.append(tpl % (value, '%s'))
values.append(filter_value) values.append(filter_value)
try: try:
if filter_values['start']: if filter_values['start']:
date_filter('%s >= %s', filter_values['start']) date_filter('%s >= %s', filter_values['start'])
@ -346,7 +361,7 @@ class Join(Base):
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.kind = 'full' self.kind = 'full'
super(Join, self).__init__(**kwargs) super().__init__(**kwargs)
@property @property
def master_table(self): def master_table(self):
@ -356,18 +371,27 @@ class Join(Base):
class Cube(Base): class Cube(Base):
__slots__ = ['name', 'label', 'fact_table', 'json_field', 'key', 'joins', 'dimensions', __slots__ = [
'measures', 'warnings'] 'name',
'label',
'fact_table',
'json_field',
'key',
'joins',
'dimensions',
'measures',
'warnings',
]
__types__ = { __types__ = {
'name': str, 'name': str,
'label': six.text_type, 'label': str,
'fact_table': str, 'fact_table': str,
'json_field': str, 'json_field': str,
'key': str, 'key': str,
'joins': [Join], 'joins': [Join],
'dimensions': [Dimension], 'dimensions': [Dimension],
'measures': [Measure], 'measures': [Measure],
'warnings': [six.text_type], 'warnings': [str],
} }
def __init__(self, **kwargs): def __init__(self, **kwargs):
@ -376,7 +400,7 @@ class Cube(Base):
self.dimensions = () self.dimensions = ()
self.measures = () self.measures = ()
self.warnings = () self.warnings = ()
super(Cube, self).__init__(**kwargs) super().__init__(**kwargs)
def check(self): def check(self):
names = collections.Counter() names = collections.Counter()
@ -386,13 +410,13 @@ class Cube(Base):
duplicates = [k for k, v in names.items() if v > 1] duplicates = [k for k, v in names.items() if v > 1]
if duplicates: if duplicates:
raise SchemaError( raise SchemaError(
'More than one join, dimension or measure with name(s) %s' % ', '.join(duplicates)) 'More than one join, dimension or measure with name(s) %s' % ', '.join(duplicates)
)
@property @property
def all_dimensions(self): def all_dimensions(self):
for dimension in self.dimensions: for dimension in self.dimensions:
for sub_dimension in dimension.dimensions: yield from dimension.dimensions
yield sub_dimension
def get_dimension(self, name): def get_dimension(self, name):
for dimension in self.dimensions: for dimension in self.dimensions:
@ -419,7 +443,7 @@ class Warehouse(Base):
__types__ = { __types__ = {
'name': str, 'name': str,
'slug': str, 'slug': str,
'label': six.text_type, 'label': str,
'pg_dsn': str, 'pg_dsn': str,
'search_path': [str], 'search_path': [str],
'cubes': [Cube], 'cubes': [Cube],
@ -430,7 +454,7 @@ class Warehouse(Base):
self.path = None self.path = None
self.slug = None self.slug = None
self.timestamp = None self.timestamp = None
super(Warehouse, self).__init__(**kwargs) super().__init__(**kwargs)
def check(self): def check(self):
names = collections.Counter(cube.name for cube in self.cubes) names = collections.Counter(cube.name for cube in self.cubes)

17
bijoe/settings.py
View File

@ -24,14 +24,14 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/ https://docs.djangoproject.com/en/1.7/ref/settings/
""" """
from itertools import chain
from django.conf import global_settings
import django
from gadjo.templatetags.gadjo import xstatic
# Build paths inside the project like this: os.path.join(BASE_DIR, ...) # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os import os
from itertools import chain
import django
from django.conf import global_settings
from gadjo.templatetags.gadjo import xstatic
BASE_DIR = os.path.dirname(os.path.dirname(__file__)) BASE_DIR = os.path.dirname(os.path.dirname(__file__))
@ -74,8 +74,7 @@ MIDDLEWARE = (
'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware',
) )
STATICFILES_FINDERS = list(chain(global_settings.STATICFILES_FINDERS, STATICFILES_FINDERS = list(chain(global_settings.STATICFILES_FINDERS, ('gadjo.finders.XStaticFinder',)))
('gadjo.finders.XStaticFinder',)))
ROOT_URLCONF = 'bijoe.urls' ROOT_URLCONF = 'bijoe.urls'
@ -122,7 +121,7 @@ LOGGING = {
'formatters': { 'formatters': {
'verbose': { 'verbose': {
'format': '[%(asctime)s] %(levelname)s %(name)s.%(funcName)s: %(message)s', 'format': '[%(asctime)s] %(levelname)s %(name)s.%(funcName)s: %(message)s',
'datefmt': '%Y-%m-%d %a %H:%M:%S' 'datefmt': '%Y-%m-%d %a %H:%M:%S',
}, },
}, },
'handlers': { 'handlers': {

2
bijoe/templatetags/bijoe.py
View File

@ -21,7 +21,7 @@ register = template.Library()
try: try:
from django_select2.templatetags.django_select2_tags import * from django_select2.templatetags.django_select2_tags import *
except ImportError: except ImportError:
@register.simple_tag(name='import_django_select2_js_css') @register.simple_tag(name='import_django_select2_js_css')
def import_all(light=0): def import_all(light=0):
return '' return ''

2
bijoe/urls.py
View File

@ -14,9 +14,9 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.conf import settings
from django.conf.urls import include, url from django.conf.urls import include, url
from django.contrib import admin from django.contrib import admin
from django.conf import settings
from . import views from . import views

14
bijoe/utils.py
View File

@ -15,14 +15,14 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime import datetime
import os
import glob import glob
import json import json
import os
from django.conf import settings from django.conf import settings
from django.db import connection, transaction from django.db import connection, transaction
from django.utils.translation import ugettext as _
from django.utils.timezone import utc from django.utils.timezone import utc
from django.utils.translation import ugettext as _
try: try:
from functools import lru_cache from functools import lru_cache
@ -34,12 +34,10 @@ from .schemas import Warehouse
def get_warehouses_paths(): def get_warehouses_paths():
for pattern in settings.BIJOE_SCHEMAS: for pattern in settings.BIJOE_SCHEMAS:
for path in glob.glob(pattern): yield from glob.glob(pattern)
yield path
if hasattr(connection, 'tenant'): if hasattr(connection, 'tenant'):
pattern = os.path.join(connection.tenant.get_directory(), 'schemas', '*.model') pattern = os.path.join(connection.tenant.get_directory(), 'schemas', '*.model')
for path in glob.glob(pattern): yield from glob.glob(pattern)
yield path
@lru_cache() @lru_cache()
@ -70,8 +68,8 @@ def human_join(l):
if len(l) == 1: if len(l) == 1:
return l[0] return l[0]
if len(l) > 2: if len(l) > 2:
l = u', '.join(l[:-1]), l[-1] l = ', '.join(l[:-1]), l[-1]
return _(u'{0} and {1}').format(l[0], l[1]) return _('{0} and {1}').format(l[0], l[1])
def export_site(): def export_site():

4
bijoe/uwsgi.py
View File

@ -21,7 +21,6 @@ import os
import subprocess import subprocess
from django.conf import settings from django.conf import settings
from uwsgidecorators import cron, spool from uwsgidecorators import cron, spool
# existing loggers are disabled by Django on django.setup() due do # existing loggers are disabled by Django on django.setup() due do
@ -58,7 +57,8 @@ def launch_wcs_olap(wcs_olap_ini_path):
'--all', '--all',
wcs_olap_ini_path, wcs_olap_ini_path,
], ],
check=False) check=False,
)
logger.info('finished wcs-olap on %s', wcs_olap_ini_path) logger.info('finished wcs-olap on %s', wcs_olap_ini_path)

29
bijoe/views.py
View File

@ -17,31 +17,31 @@
import json import json
from django.conf import settings from django.conf import settings
from django.shortcuts import resolve_url
from django.urls import reverse
from django.views.generic import ListView, View
from django.http import HttpResponse, HttpResponseRedirect
from django.utils.decorators import method_decorator
from django.utils.http import quote
from django.utils.translation import ugettext as _
from django.contrib.auth import logout as auth_logout from django.contrib.auth import logout as auth_logout
from django.contrib.auth import views as auth_views from django.contrib.auth import views as auth_views
from django.contrib.auth.views import redirect_to_login from django.contrib.auth.views import redirect_to_login
from django.core.exceptions import PermissionDenied from django.core.exceptions import PermissionDenied
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import resolve_url
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.utils.http import quote
from django.utils.translation import ugettext as _
from django.views.decorators.cache import never_cache from django.views.decorators.cache import never_cache
from django.views.generic import ListView, View
try: try:
from mellon.utils import get_idps from mellon.utils import get_idps
except ImportError: except ImportError:
get_idps = lambda: [] get_idps = lambda: []
from .utils import get_warehouses
from .engine import Engine from .engine import Engine
from .utils import get_warehouses
from .visualization.models import Visualization from .visualization.models import Visualization
from .visualization.utils import Visualization as VisuUtil from .visualization.utils import Visualization as VisuUtil
class AuthorizationMixin(object): class AuthorizationMixin:
def authorize(self, request): def authorize(self, request):
if request.user.is_authenticated: if request.user.is_authenticated:
if not request.user.is_superuser: if not request.user.is_superuser:
@ -52,7 +52,7 @@ class AuthorizationMixin(object):
def dispatch(self, request, *args, **kwargs): def dispatch(self, request, *args, **kwargs):
if self.authorize(request): if self.authorize(request):
return super(AuthorizationMixin, self).dispatch(request, *args, **kwargs) return super().dispatch(request, *args, **kwargs)
else: else:
return redirect_to_login(request.build_absolute_uri()) return redirect_to_login(request.build_absolute_uri())
@ -64,9 +64,8 @@ class HomepageView(AuthorizationMixin, ListView):
paginate_by = settings.PAGE_LENGTH paginate_by = settings.PAGE_LENGTH
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
ctx = super(HomepageView, self).get_context_data(**kwargs) ctx = super().get_context_data(**kwargs)
ctx['warehouses'] = sorted((Engine(w) for w in get_warehouses()), ctx['warehouses'] = sorted((Engine(w) for w in get_warehouses()), key=lambda w: w.label)
key=lambda w: w.label)
ctx['request'] = self.request ctx['request'] = self.request
return ctx return ctx
@ -108,7 +107,7 @@ class LoginView(auth_views.LoginView):
return HttpResponseRedirect( return HttpResponseRedirect(
resolve_url('mellon_login') + '?next=' + quote(request.GET.get('next')) resolve_url('mellon_login') + '?next=' + quote(request.GET.get('next'))
) )
return super(LoginView, self).get(request, *args, **kwargs) return super().get(request, *args, **kwargs)
login = LoginView.as_view() login = LoginView.as_view()
@ -119,7 +118,7 @@ class LogoutView(auth_views.LogoutView):
def dispatch(self, request, *args, **kwargs): def dispatch(self, request, *args, **kwargs):
if any(get_idps()): if any(get_idps()):
return HttpResponseRedirect(resolve_url('mellon_logout')) return HttpResponseRedirect(resolve_url('mellon_logout'))
return super(LogoutView, self).dispatch(request, *args, **kwargs) return super().dispatch(request, *args, **kwargs)
logout = LogoutView.as_view() logout = LogoutView.as_view()

1
bijoe/visualization/admin.py
View File

@ -22,4 +22,5 @@ from . import models
class VisualizationAdmin(admin.ModelAdmin): class VisualizationAdmin(admin.ModelAdmin):
list_display = ['name'] list_display = ['name']
admin.site.register(models.Visualization, VisualizationAdmin) admin.site.register(models.Visualization, VisualizationAdmin)

113
bijoe/visualization/forms.py
View File

@ -1,4 +1,3 @@
# -*- encoding: utf-8 -*-
# bijoe - BI dashboard # bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert # Copyright (C) 2015 Entr'ouvert
# #
@ -17,13 +16,12 @@
from django import forms from django import forms
from django.core.exceptions import ValidationError
from django.utils.encoding import force_text
from django.utils.translation import ugettext as _
from django.utils.safestring import mark_safe
from django.forms import ModelForm, TextInput, NullBooleanField
from django.conf import settings from django.conf import settings
from django.core.exceptions import ValidationError
from django.forms import ModelForm, NullBooleanField, TextInput
from django.utils.encoding import force_text
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _
from django_select2.forms import HeavySelect2MultipleWidget from django_select2.forms import HeavySelect2MultipleWidget
from . import models from . import models
@ -32,54 +30,59 @@ from . import models
class VisualizationForm(ModelForm): class VisualizationForm(ModelForm):
class Meta: class Meta:
model = models.Visualization model = models.Visualization
exclude = ('slug', 'parameters',) exclude = (
'slug',
'parameters',
)
widgets = { widgets = {
'name': TextInput, 'name': TextInput,
} }
DATE_RANGES = [ DATE_RANGES = [
{ {
'value': '3_last_months', 'value': '3_last_months',
'label': _('3 last months'), 'label': _('3 last months'),
'start': u"les 3 derniers mois", 'start': "les 3 derniers mois",
'end': u"maintenant", 'end': "maintenant",
}, },
{ {
'value': 'this_year', 'value': 'this_year',
'label': _('this year'), 'label': _('this year'),
'start': u"cette année", 'start': "cette année",
'end': u"l\'année prochaine", 'end': "l\'année prochaine",
}, },
{ {
'value': 'last_year', 'value': 'last_year',
'label': _('last year'), 'label': _('last year'),
'start': u'l\'année dernière', 'start': 'l\'année dernière',
'end': u'cette année', 'end': 'cette année',
}, },
{ {
'value': 'this_quarter', 'value': 'this_quarter',
'label': _('this quarter'), 'label': _('this quarter'),
'start': u'ce trimestre', 'start': 'ce trimestre',
'end': "le prochain trimestre", 'end': "le prochain trimestre",
}, },
{ {
'value': 'last_quarter', 'value': 'last_quarter',
'label': _('last quarter'), 'label': _('last quarter'),
'start': u'le dernier trimestre', 'start': 'le dernier trimestre',
'end': u'ce trimestre', 'end': 'ce trimestre',
}, },
{ {
'value': 'since_1jan_last_year', 'value': 'since_1jan_last_year',
'label': _('since 1st january last year'), 'label': _('since 1st january last year'),
'start': u'l\'année dernière', 'start': 'l\'année dernière',
'end': u'maintenant', 'end': 'maintenant',
}, },
] ]
def get_date_range_choices(): def get_date_range_choices():
return [('', '---')] + [(r['value'], r['label']) return [('', '---')] + [
for r in getattr(settings, 'BIJOE_DATE_RANGES', DATE_RANGES)] (r['value'], r['label']) for r in getattr(settings, 'BIJOE_DATE_RANGES', DATE_RANGES)
]
class DateRangeWidget(forms.MultiWidget): class DateRangeWidget(forms.MultiWidget):
@ -87,15 +90,15 @@ class DateRangeWidget(forms.MultiWidget):
attrs = attrs.copy() if attrs else {} attrs = attrs.copy() if attrs else {}
attrs.update({'type': 'date', 'autocomplete': 'off'}) attrs.update({'type': 'date', 'autocomplete': 'off'})
attrs1 = attrs.copy() attrs1 = attrs.copy()
attrs1['placeholder'] = _(u'start') attrs1['placeholder'] = _('start')
attrs2 = attrs.copy() attrs2 = attrs.copy()
attrs2['placeholder'] = _(u'end') attrs2['placeholder'] = _('end')
widgets = ( widgets = (
forms.DateInput(attrs=attrs1, format='%Y-%m-%d'), forms.DateInput(attrs=attrs1, format='%Y-%m-%d'),
forms.DateInput(attrs=attrs2, format='%Y-%m-%d'), forms.DateInput(attrs=attrs2, format='%Y-%m-%d'),
forms.Select(choices=get_date_range_choices()), forms.Select(choices=get_date_range_choices()),
) )
super(DateRangeWidget, self).__init__(widgets, attrs=attrs) super().__init__(widgets, attrs=attrs)
def decompress(self, value): def decompress(self, value):
if not value: if not value:
@ -106,11 +109,10 @@ class DateRangeWidget(forms.MultiWidget):
return value['start'], value['end'], None return value['start'], value['end'], None
def render(self, name, value, attrs=None, renderer=None): def render(self, name, value, attrs=None, renderer=None):
output = super(DateRangeWidget, self).render(name, value, attrs=attrs) output = super().render(name, value, attrs=attrs)
_id = self.build_attrs(attrs).get('id', None) _id = self.build_attrs(attrs).get('id', None)
if _id: if _id:
output += mark_safe("<script>$(function () { bijoe_date_range('#%s'); });</script>" % output += mark_safe("<script>$(function () { bijoe_date_range('#%s'); });</script>" % _id)
_id)
return output return output
class Media: class Media:
@ -125,10 +127,9 @@ class DateRangeField(forms.MultiValueField):
fields = ( fields = (
forms.DateField(required=False), forms.DateField(required=False),
forms.DateField(required=False), forms.DateField(required=False),
forms.ChoiceField(choices=get_date_range_choices(), required=False) forms.ChoiceField(choices=get_date_range_choices(), required=False),
) )
super(DateRangeField, self).__init__(fields=fields, require_all_fields=False, *args, super().__init__(fields=fields, require_all_fields=False, *args, **kwargs)
**kwargs)
def compress(self, values): def compress(self, values):
if not values: if not values:
@ -159,10 +160,10 @@ class Select2ChoicesWidget(HeavySelect2MultipleWidget):
class CubeForm(forms.Form): class CubeForm(forms.Form):
representation = forms.ChoiceField( representation = forms.ChoiceField(
label=_(u'Presentation'), label=_('Presentation'),
choices=[('table', _('table')), choices=[('table', _('table')), ('graphical', _('chart'))],
('graphical', _('chart'))], widget=forms.RadioSelect(),
widget=forms.RadioSelect()) )
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.cube = cube = kwargs.pop('cube') self.cube = cube = kwargs.pop('cube')
@ -170,12 +171,13 @@ class CubeForm(forms.Form):
dimension_choices = [('', '')] + [ dimension_choices = [('', '')] + [
(dimension.name, dimension.label) (dimension.name, dimension.label)
for dimension in cube.dimensions if dimension.type not in ('datetime', 'date')] for dimension in cube.dimensions
if dimension.type not in ('datetime', 'date')
]
# loop # loop
self.base_fields['loop'] = forms.ChoiceField( self.base_fields['loop'] = forms.ChoiceField(
label=_('Loop by'), label=_('Loop by'), choices=dimension_choices, required=False
choices=dimension_choices, )
required=False)
# filters # filters
for dimension in cube.dimensions: for dimension in cube.dimensions:
@ -184,10 +186,12 @@ class CubeForm(forms.Form):
field_name = 'filter__%s' % dimension.name field_name = 'filter__%s' % dimension.name
if dimension.type == 'date': if dimension.type == 'date':
self.base_fields[field_name] = DateRangeField( self.base_fields[field_name] = DateRangeField(
label=dimension.label.capitalize(), required=False) label=dimension.label.capitalize(), required=False
)
elif dimension.type == 'bool': elif dimension.type == 'bool':
self.base_fields[field_name] = NullBooleanField( self.base_fields[field_name] = NullBooleanField(
label=dimension.label.capitalize(), required=False) label=dimension.label.capitalize(), required=False
)
else: else:
members = [] members = []
for _id, label in dimension.members(): for _id, label in dimension.members():
@ -200,6 +204,7 @@ class CubeForm(forms.Form):
if v == s: if v == s:
return value return value
return None return None
return f return f
self.base_fields[field_name] = forms.TypedMultipleChoiceField( self.base_fields[field_name] = forms.TypedMultipleChoiceField(
@ -211,38 +216,34 @@ class CubeForm(forms.Form):
data_view='select2-choices', data_view='select2-choices',
warehouse=cube.engine.warehouse.name, warehouse=cube.engine.warehouse.name,
cube=cube.name, cube=cube.name,
dimension=dimension.name dimension=dimension.name,
)) ),
)
# group by # group by
self.base_fields['drilldown_x'] = forms.ChoiceField( self.base_fields['drilldown_x'] = forms.ChoiceField(
label=_('Group by - horizontally'), label=_('Group by - horizontally'), choices=dimension_choices, required=False
choices=dimension_choices, )
required=False)
self.base_fields['drilldown_y'] = forms.ChoiceField( self.base_fields['drilldown_y'] = forms.ChoiceField(
label=_('Group by - vertically'), label=_('Group by - vertically'), choices=dimension_choices, required=False
choices=dimension_choices, )
required=False)
# measures # measures
choices = [(measure.name, measure.label) choices = [(measure.name, measure.label) for measure in cube.measures if measure.type != 'point']
for measure in cube.measures if measure.type != 'point'] self.base_fields['measure'] = forms.ChoiceField(label=_('Measure'), choices=choices)
self.base_fields['measure'] = forms.ChoiceField(
label=_('Measure'), choices=choices)
super(CubeForm, self).__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def clean(self): def clean(self):
cleaned_data = super(CubeForm, self).clean() cleaned_data = super().clean()
loop = cleaned_data.get('loop') loop = cleaned_data.get('loop')
drilldown_x = cleaned_data.get('drilldown_x') drilldown_x = cleaned_data.get('drilldown_x')
drilldown_y = cleaned_data.get('drilldown_y') drilldown_y = cleaned_data.get('drilldown_y')
if loop and (loop == drilldown_x or loop == drilldown_y): if loop and (loop == drilldown_x or loop == drilldown_y):
raise ValidationError({'loop': _('You cannot use the same dimension for looping and' raise ValidationError({'loop': _('You cannot use the same dimension for looping and grouping')})
' grouping')})
return cleaned_data return cleaned_data

13
bijoe/visualization/migrations/0001_initial.py
View File

@ -1,20 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import jsonfield.fields import jsonfield.fields
from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = []
]
operations = [ operations = [
migrations.CreateModel( migrations.CreateModel(
name='Visualization', name='Visualization',
fields=[ fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), (
'id',
models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True),
),
('name', models.TextField(verbose_name='name')), ('name', models.TextField(verbose_name='name')),
('parameters', jsonfield.fields.JSONField(default=dict, verbose_name='parameters')), ('parameters', jsonfield.fields.JSONField(default=dict, verbose_name='parameters')),
], ],

3
bijoe/visualization/migrations/0002_rename_parameters.py
View File

@ -1,6 +1,3 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations from django.db import migrations

4
bijoe/visualization/migrations/0003_visualization_slug.py
View File

@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.12 on 2019-03-28 07:17 # Generated by Django 1.11.12 on 2019-03-28 07:17
from __future__ import unicode_literals
from django.db import migrations, models from django.db import migrations, models
from django.utils.text import slugify from django.utils.text import slugify
@ -38,5 +36,5 @@ class Migration(migrations.Migration):
name='slug', name='slug',
field=models.SlugField(null=True, unique=True, verbose_name='Identifier'), field=models.SlugField(null=True, unique=True, verbose_name='Identifier'),
), ),
migrations.RunPython(forward_func, reverse_func) migrations.RunPython(forward_func, reverse_func),
] ]

2
bijoe/visualization/migrations/0004_auto_20190328_0825.py
View File

@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.12 on 2019-03-28 07:25 # Generated by Django 1.11.12 on 2019-03-28 07:25
from __future__ import unicode_literals
from django.db import migrations, models from django.db import migrations, models

15
bijoe/visualization/models.py
View File

@ -14,8 +14,8 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import datetime import datetime
import json
from django.db import models from django.db import models
from django.http import Http404 from django.http import Http404
@ -57,19 +57,12 @@ class Visualization(models.Model):
return (self.slug,) return (self.slug,)
def export_json(self): def export_json(self):
visualization = { visualization = {'slug': self.slug, 'name': self.name, 'parameters': self.parameters}
'slug': self.slug,
'name': self.name,
'parameters': self.parameters
}
return visualization return visualization
@classmethod @classmethod
def import_json(cls, data): def import_json(cls, data):
defaults = { defaults = {'name': data['name'], 'parameters': data['parameters']}
'name': data['name'],
'parameters': data['parameters']
}
_, created = cls.objects.update_or_create(slug=data['slug'], defaults=defaults) _, created = cls.objects.update_or_create(slug=data['slug'], defaults=defaults)
return created return created
@ -85,7 +78,7 @@ class Visualization(models.Model):
i += 1 i += 1
slug = '%s-%s' % (base_slug, i) slug = '%s-%s' % (base_slug, i)
self.slug = slug self.slug = slug
return super(Visualization, self).save(*args, **kwargs) return super().save(*args, **kwargs)
@property @property
def exists(self): def exists(self):

30
bijoe/visualization/ods.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# #
# bijoe - BI dashboard # bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert # Copyright (C) 2015 Entr'ouvert
@ -17,13 +16,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys import sys
import zipfile
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import zipfile
from django.utils.encoding import force_text from django.utils.encoding import force_text
OFFICE_NS = 'urn:oasis:names:tc:opendocument:xmlns:office:1.0' OFFICE_NS = 'urn:oasis:names:tc:opendocument:xmlns:office:1.0'
TABLE_NS = 'urn:oasis:names:tc:opendocument:xmlns:table:1.0' TABLE_NS = 'urn:oasis:names:tc:opendocument:xmlns:table:1.0'
TEXT_NS = 'urn:oasis:names:tc:opendocument:xmlns:text:1.0' TEXT_NS = 'urn:oasis:names:tc:opendocument:xmlns:text:1.0'
@ -31,13 +28,10 @@ XLINK_NS = 'http://www.w3.org/1999/xlink'
def is_number(x): def is_number(x):
if sys.version_info >= (3, 0): return isinstance(x, (int, float))
return isinstance(x, (int, float))
else:
return isinstance(x, (int, long, float))
class Workbook(object): class Workbook:
def __init__(self, encoding='utf-8'): def __init__(self, encoding='utf-8'):
self.sheets = [] self.sheets = []
self.encoding = encoding self.encoding = encoding
@ -64,21 +58,27 @@ class Workbook(object):
z = zipfile.ZipFile(output, 'w') z = zipfile.ZipFile(output, 'w')
z.writestr('content.xml', self.get_data()) z.writestr('content.xml', self.get_data())
z.writestr('mimetype', 'application/vnd.oasis.opendocument.spreadsheet') z.writestr('mimetype', 'application/vnd.oasis.opendocument.spreadsheet')
z.writestr('META-INF/manifest.xml', '''<?xml version="1.0" encoding="UTF-8"?> z.writestr(
'META-INF/manifest.xml',
'''<?xml version="1.0" encoding="UTF-8"?>
<manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0"> <manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0">
<manifest:file-entry manifest:full-path="/" manifest:media-type="application/vnd.oasis.opendocument.spreadsheet"/> <manifest:file-entry manifest:full-path="/" manifest:media-type="application/vnd.oasis.opendocument.spreadsheet"/>
<manifest:file-entry manifest:full-path="styles.xml" manifest:media-type="text/xml"/> <manifest:file-entry manifest:full-path="styles.xml" manifest:media-type="text/xml"/>
<manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/> <manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/>
<manifest:file-entry manifest:full-path="META-INF/manifest.xml" manifest:media-type="text/xml"/> <manifest:file-entry manifest:full-path="META-INF/manifest.xml" manifest:media-type="text/xml"/>
<manifest:file-entry manifest:full-path="mimetype" manifest:media-type="text/plain"/> <manifest:file-entry manifest:full-path="mimetype" manifest:media-type="text/plain"/>
</manifest:manifest>''') </manifest:manifest>''',
z.writestr('styles.xml', '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?> )
z.writestr(
'styles.xml',
'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<office:document-styles xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"> <office:document-styles xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0">
</office:document-styles>''') </office:document-styles>''',
)
z.close() z.close()
class WorkSheet(object): class WorkSheet:
def __init__(self, workbook, name): def __init__(self, workbook, name):
self.cells = {} self.cells = {}
self.name = name self.name = name
@ -104,7 +104,7 @@ class WorkSheet(object):
return root return root
class WorkCell(object): class WorkCell:
def __init__(self, worksheet, value, hint=None): def __init__(self, worksheet, value, hint=None):
self.value_type = 'string' self.value_type = 'string'
if is_number(value): if is_number(value):

24
bijoe/visualization/signature.py
View File

@ -14,13 +14,13 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
import base64 import base64
import hmac import datetime
import hashlib import hashlib
import urllib import hmac
import random
import logging import logging
import random
import urllib
from django.utils import six from django.utils import six
from django.utils.encoding import force_bytes, smart_bytes from django.utils.encoding import force_bytes, smart_bytes
@ -46,10 +46,7 @@ def sign_query(query, key, algo='sha256', timestamp=None, nonce=None):
new_query = query new_query = query
if new_query: if new_query:
new_query += '&' new_query += '&'
new_query += urlencode(( new_query += urlencode((('algo', algo), ('timestamp', timestamp), ('nonce', nonce)))
('algo', algo),
('timestamp', timestamp),
('nonce', nonce)))
signature = base64.b64encode(sign_string(new_query, key, algo=algo)) signature = base64.b64encode(sign_string(new_query, key, algo=algo))
new_query += '&signature=' + quote(signature) new_query += '&signature=' + quote(signature)
return new_query return new_query
@ -71,7 +68,8 @@ def check_query(query, key, known_nonce=None, timedelta=30):
if not res: if not res:
key_hash = 'md5:%s' % hashlib.md5(force_bytes(key)).hexdigest()[:6] key_hash = 'md5:%s' % hashlib.md5(force_bytes(key)).hexdigest()[:6]
logging.getLogger(__name__).warning( logging.getLogger(__name__).warning(
'could not check signature of query %r with key %s: %s', query, key_hash, error) 'could not check signature of query %r with key %s: %s', query, key_hash, error
)
return res return res
@ -112,10 +110,6 @@ def check_string(s, signature, key, algo='sha256'):
if len(signature2) != len(signature): if len(signature2) != len(signature):
return False return False
res = 0 res = 0
if six.PY3: for a, b in zip(signature, signature2):
for a, b in zip(signature, signature2): res |= a ^ b
res |= a ^ b
else:
for a, b in zip(signature, signature2):
res |= ord(a) ^ ord(b)
return res == 0 return res == 0

22
bijoe/visualization/urls.py
View File

@ -19,20 +19,18 @@ from django.conf.urls import url
from . import views from . import views
urlpatterns = [ urlpatterns = [
url(r'^$', url(r'^$', views.visualizations, name='visualizations'),
views.visualizations, name='visualizations'), url(r'^json/$', views.visualizations_json, name='visualizations-json'),
url(r'^json/$', url(r'^import/$', views.visualizations_import, name='visualizations-import'),
views.visualizations_json, name='visualizations-json'), url(r'^export$', views.visualizations_export, name='visualizations-export'),
url(r'^import/$',
views.visualizations_import, name='visualizations-import'),
url(r'^export$',
views.visualizations_export, name='visualizations-export'),
url(r'^warehouse/(?P<warehouse>[^/]*)/$', views.warehouse, name='warehouse'), url(r'^warehouse/(?P<warehouse>[^/]*)/$', views.warehouse, name='warehouse'),
url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/$', views.cube, name='cube'), url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/$', views.cube, name='cube'),
url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/iframe/$', views.cube_iframe, url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/iframe/$', views.cube_iframe, name='cube-iframe'),
name='cube-iframe'), url(
url(r'warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/save/$', r'warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/save/$',
views.create_visualization, name='create-visualization'), views.create_visualization,
name='create-visualization',
),
url(r'(?P<pk>\d+)/$', views.visualization, name='visualization'), url(r'(?P<pk>\d+)/$', views.visualization, name='visualization'),
url(r'(?P<pk>\d+)/json/$', views.visualization_json, name='visualization-json'), url(r'(?P<pk>\d+)/json/$', views.visualization_json, name='visualization-json'),
url(r'(?P<pk>\d+)/geojson/$', views.visualization_geojson, name='visualization-geojson'), url(r'(?P<pk>\d+)/geojson/$', views.visualization_geojson, name='visualization-geojson'),

108
bijoe/visualization/utils.py
View File

@ -14,31 +14,31 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import re import collections
import json import copy
import hashlib
import datetime import datetime
import decimal import decimal
import copy import hashlib
import collections import json
import re
from django.conf import settings
from django.core.cache import cache from django.core.cache import cache
from django.http import Http404
from django.utils.encoding import force_bytes, force_text from django.utils.encoding import force_bytes, force_text
from django.utils.safestring import mark_safe from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from django.http import Http404
from django.conf import settings
from ..engine import Engine, MeasureCell, Member
from ..utils import get_warehouses from ..utils import get_warehouses
from ..engine import Engine, Member, MeasureCell
from .ods import Workbook from .ods import Workbook
class Visualization(object): class Visualization:
def __init__(self, cube, representation, measure, drilldown_x=None, drilldown_y=None, def __init__(
filters=None, loop=None): self, cube, representation, measure, drilldown_x=None, drilldown_y=None, filters=None, loop=None
):
self.cube = cube self.cube = cube
self.representation = representation self.representation = representation
@ -74,9 +74,15 @@ class Visualization(object):
} }
def copy(self): def copy(self):
return Visualization(self.cube, self.representation, measure=self.measure, return Visualization(
drilldown_x=self.drilldown_x, drilldown_y=self.drilldown_y, self.cube,
filters=copy.deepcopy(self.filters), loop=self.loop) self.representation,
measure=self.measure,
drilldown_x=self.drilldown_x,
drilldown_y=self.drilldown_y,
filters=copy.deepcopy(self.filters),
loop=self.loop,
)
@staticmethod @staticmethod
def get_cube(d, warehouses=None): def get_cube(d, warehouses=None):
@ -107,8 +113,15 @@ class Visualization(object):
loop = d.get('loop') loop = d.get('loop')
if loop: if loop:
loop = cube.dimensions[loop] loop = cube.dimensions[loop]
return cls(cube, representation, measure, drilldown_x=drilldown_x, drilldown_y=drilldown_y, return cls(
filters=filters, loop=loop) cube,
representation,
measure,
drilldown_x=drilldown_x,
drilldown_y=drilldown_y,
filters=filters,
loop=loop,
)
@classmethod @classmethod
def from_form(cls, cube, form): def from_form(cls, cube, form):
@ -126,11 +139,15 @@ class Visualization(object):
drilldown_y = drilldown_y and cube.dimensions[drilldown_y] drilldown_y = drilldown_y and cube.dimensions[drilldown_y]
loop = cleaned_data.get('loop') loop = cleaned_data.get('loop')
loop = loop and cube.dimensions[loop] loop = loop and cube.dimensions[loop]
return cls(cube, cleaned_data['representation'], return cls(
measure, cube,
drilldown_x=drilldown_x, cleaned_data['representation'],
drilldown_y=drilldown_y, measure,
filters=filters, loop=loop) drilldown_x=drilldown_x,
drilldown_y=drilldown_y,
filters=filters,
loop=loop,
)
@property @property
def key(self): def key(self):
@ -147,21 +164,20 @@ class Visualization(object):
keys.append('$'.join([kw] + sorted(map(force_text, value)))) keys.append('$'.join([kw] + sorted(map(force_text, value))))
else: else:
# scalar values # scalar values
keys.append(u'%s$%s' % (kw, force_text(value))) keys.append('%s$%s' % (kw, force_text(value)))
keys += [dim.name for dim in self.drilldown] keys += [dim.name for dim in self.drilldown]
keys += [self.measure.name] keys += [self.measure.name]
key = '$'.join(v.encode('utf8') for v in keys) key = '$'.join(v.encode('utf8') for v in keys)
return hashlib.md5(force_bytes(key)).hexdigest() return hashlib.md5(force_bytes(key)).hexdigest()
def data(self): def data(self):
'''Execute aggregation query, list members and check None values in """Execute aggregation query, list members and check None values in
dimensions. dimensions.
''' """
rows = list(self.cube.query(self.filters.items(), rows = list(self.cube.query(self.filters.items(), self.drilldown, [self.measure]))
self.drilldown, self.members = {
[self.measure])) dimension: list(dimension.members(filters=self.filters.items())) for dimension in self.drilldown
self.members = {dimension: list(dimension.members(filters=self.filters.items())) }
for dimension in self.drilldown}
seen_none = set() seen_none = set()
for cells in rows: for cells in rows:
# Keep "empty" dimension value if there is a non-zero measure associated # Keep "empty" dimension value if there is a non-zero measure associated
@ -268,10 +284,12 @@ class Visualization(object):
y_axis, grid = self.table_1d() y_axis, grid = self.table_1d()
table.append([self.drilldown_y.label, self.measure.label]) table.append([self.drilldown_y.label, self.measure.label])
for y in y_axis: for y in y_axis:
table.append([ table.append(
y.label, [
'%s' % (grid[y.id],), y.label,
]) '%s' % (grid[y.id],),
]
)
else: else:
table.append([self.measure.label, '%s' % (self.data()[0].measures[0],)]) table.append([self.measure.label, '%s' % (self.data()[0].measures[0],)])
@ -296,12 +314,16 @@ class Visualization(object):
if isinstance(value, decimal.Decimal): if isinstance(value, decimal.Decimal):
value = float(value) value = float(value)
if isinstance(value, datetime.timedelta): if isinstance(value, datetime.timedelta):
value = value.days + value.seconds / 86400. value = value.days + value.seconds / 86400.0
return value return value
if len(self.drilldown) == 2: if len(self.drilldown) == 2:
(x_axis, y_axis), grid = self.table_2d() (x_axis, y_axis), grid = self.table_2d()
cells = ((['%s' % x.label, '%s' % y.label], cell_value(grid[(x.id, y.id)])) for x in x_axis for y in y_axis) cells = (
(['%s' % x.label, '%s' % y.label], cell_value(grid[(x.id, y.id)]))
for x in x_axis
for y in y_axis
)
elif len(self.drilldown) == 1: elif len(self.drilldown) == 1:
axis, grid = self.table_1d() axis, grid = self.table_1d()
cells = ((['%s' % x.label], cell_value(grid[x.id])) for x in axis) cells = ((['%s' % x.label], cell_value(grid[x.id])) for x in axis)
@ -315,10 +337,12 @@ class Visualization(object):
raise NotImplementedError raise NotImplementedError
for coords, value in cells: for coords, value in cells:
json_data.append({ json_data.append(
'coords': [{'value': coord} for coord in coords], {
'measures': [{'value': value}], 'coords': [{'value': coord} for coord in coords],
}) 'measures': [{'value': value}],
}
)
return json_data return json_data
@ -350,7 +374,7 @@ class Visualization(object):
l.append(self.drilldown_y.label) l.append(self.drilldown_y.label)
if self.loop: if self.loop:
l.append(self.loop.label) l.append(self.loop.label)
return u', '.join(l) return ', '.join(l)
def __iter__(self): def __iter__(self):
if self.loop: if self.loop:

177
bijoe/visualization/views.py
View File

@ -14,47 +14,48 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import hashlib import hashlib
import json import json
from django.conf import settings from django.conf import settings
from django.core import signing
from django.core.signing import BadSignature
from django.contrib import messages from django.contrib import messages
from django.core import signing
from django.core.exceptions import PermissionDenied
from django.core.signing import BadSignature
from django.http import Http404, HttpResponse, JsonResponse
from django.shortcuts import redirect
from django.urls import reverse, reverse_lazy
from django.utils.encoding import force_bytes, force_text from django.utils.encoding import force_bytes, force_text
from django.utils.text import slugify from django.utils.text import slugify
from django.utils.timezone import now from django.utils.timezone import now
from django.utils.translation import ungettext, ugettext as _ from django.utils.translation import ugettext as _
from django.views.generic.edit import CreateView, DeleteView, UpdateView, FormView from django.utils.translation import ungettext
from django.views.generic.list import MultipleObjectMixin
from django.views.generic import DetailView, ListView, View, TemplateView
from django.shortcuts import redirect
from django.urls import reverse, reverse_lazy
from django.http import HttpResponse, Http404, JsonResponse
from django.core.exceptions import PermissionDenied
from django.views.decorators.clickjacking import xframe_options_exempt from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.generic import DetailView, ListView, TemplateView, View
from django.views.generic.edit import CreateView, DeleteView, FormView, UpdateView
from django.views.generic.list import MultipleObjectMixin
from django_select2.cache import cache from django_select2.cache import cache
from rest_framework import generics from rest_framework import generics
from rest_framework.response import Response from rest_framework.response import Response
from bijoe.utils import get_warehouses, import_site, export_site from bijoe.utils import export_site, get_warehouses, import_site
from ..engine import Engine
from . import models, forms, signature
from .utils import Visualization
from .. import views from .. import views
from ..engine import Engine
from . import forms, models, signature
from .utils import Visualization
class WarehouseView(views.AuthorizationMixin, TemplateView): class WarehouseView(views.AuthorizationMixin, TemplateView):
template_name = 'bijoe/warehouse.html' template_name = 'bijoe/warehouse.html'
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
ctx = super(WarehouseView, self).get_context_data(**kwargs) ctx = super().get_context_data(**kwargs)
try: try:
warehouse = [warehouse for warehouse in get_warehouses() warehouse = [
if warehouse.name == self.kwargs['warehouse']][0] warehouse for warehouse in get_warehouses() if warehouse.name == self.kwargs['warehouse']
][0]
except IndexError: except IndexError:
raise Http404 raise Http404
ctx['warehouse'] = Engine(warehouse) ctx['warehouse'] = Engine(warehouse)
@ -63,16 +64,16 @@ class WarehouseView(views.AuthorizationMixin, TemplateView):
return ctx return ctx
class CubeDisplayMixin(object): class CubeDisplayMixin:
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
ctx = super(CubeDisplayMixin, self).get_context_data(**kwargs) ctx = super().get_context_data(**kwargs)
ctx['warehouse'] = self.warehouse ctx['warehouse'] = self.warehouse
ctx['cube'] = self.cube ctx['cube'] = self.cube
ctx['visualization'] = self.visualization ctx['visualization'] = self.visualization
return ctx return ctx
class CubeMixin(object): class CubeMixin:
def visualization(self, request, cube): def visualization(self, request, cube):
self.form = forms.CubeForm(cube=self.cube, data=request.GET or request.POST) self.form = forms.CubeForm(cube=self.cube, data=request.GET or request.POST)
if self.form.is_valid(): if self.form.is_valid():
@ -80,8 +81,9 @@ class CubeMixin(object):
def dispatch(self, request, *args, **kwargs): def dispatch(self, request, *args, **kwargs):
try: try:
self.warehouse = Engine([warehouse for warehouse in get_warehouses() self.warehouse = Engine(
if warehouse.name == self.kwargs['warehouse']][0]) [warehouse for warehouse in get_warehouses() if warehouse.name == self.kwargs['warehouse']][0]
)
except IndexError: except IndexError:
raise Http404 raise Http404
try: try:
@ -89,7 +91,7 @@ class CubeMixin(object):
except KeyError: except KeyError:
raise Http404 raise Http404
self.visualization = self.visualization(request, cube) self.visualization = self.visualization(request, cube)
return super(CubeMixin, self).dispatch(request, *args, **kwargs) return super().dispatch(request, *args, **kwargs)
class CubeView(views.AuthorizationMixin, CubeDisplayMixin, CubeMixin, TemplateView): class CubeView(views.AuthorizationMixin, CubeDisplayMixin, CubeMixin, TemplateView):
@ -101,7 +103,7 @@ class CubeView(views.AuthorizationMixin, CubeDisplayMixin, CubeMixin, TemplateVi
return self.get(request, *args, **kwargs) return self.get(request, *args, **kwargs)
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
ctx = super(CubeView, self).get_context_data(**kwargs) ctx = super().get_context_data(**kwargs)
ctx['form'] = self.form ctx['form'] = self.form
return ctx return ctx
@ -115,13 +117,13 @@ class CreateVisualizationView(views.AuthorizationMixin, CubeMixin, CreateView):
def get(self, request, *args, **kwargs): def get(self, request, *args, **kwargs):
if not self.visualization: if not self.visualization:
return redirect('homepage') return redirect('homepage')
return super(CreateVisualizationView, self).get(request, *args, **kwargs) return super().get(request, *args, **kwargs)
def form_valid(self, form): def form_valid(self, form):
if not self.visualization: if not self.visualization:
return redirect('homepage') return redirect('homepage')
form.instance.parameters = self.visualization.to_json() form.instance.parameters = self.visualization.to_json()
return super(CreateVisualizationView, self).form_valid(form) return super().form_valid(form)
class SaveAsVisualizationView(views.AuthorizationMixin, DetailView, CreateView): class SaveAsVisualizationView(views.AuthorizationMixin, DetailView, CreateView):
@ -132,12 +134,10 @@ class SaveAsVisualizationView(views.AuthorizationMixin, DetailView, CreateView):
def form_valid(self, form): def form_valid(self, form):
form.instance.parameters = self.get_object().parameters form.instance.parameters = self.get_object().parameters
return super(SaveAsVisualizationView, self).form_valid(form) return super().form_valid(form)
def get_initial(self): def get_initial(self):
return { return {'name': '%s %s' % (self.get_object().name, _('(Copy)'))}
'name': '%s %s' % (self.get_object().name, _('(Copy)'))
}
class VisualizationView(views.AuthorizationMixin, CubeDisplayMixin, DetailView): class VisualizationView(views.AuthorizationMixin, CubeDisplayMixin, DetailView):
@ -145,7 +145,7 @@ class VisualizationView(views.AuthorizationMixin, CubeDisplayMixin, DetailView):
template_name = 'bijoe/visualization.html' template_name = 'bijoe/visualization.html'
def get_object(self): def get_object(self):
named_visualization = super(VisualizationView, self).get_object() named_visualization = super().get_object()
if not hasattr(self, 'visualization'): if not hasattr(self, 'visualization'):
self.visualization = Visualization.from_json(named_visualization.parameters) self.visualization = Visualization.from_json(named_visualization.parameters)
self.cube = self.visualization.cube self.cube = self.visualization.cube
@ -165,7 +165,7 @@ class VisualizationView(views.AuthorizationMixin, CubeDisplayMixin, DetailView):
return self.get(request, *args, **kwargs) return self.get(request, *args, **kwargs)
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
ctx = super(VisualizationView, self).get_context_data(**kwargs) ctx = super().get_context_data(**kwargs)
initial = { initial = {
'representation': self.visualization.representation, 'representation': self.visualization.representation,
'measure': self.visualization.measure.name, 'measure': self.visualization.measure.name,
@ -218,7 +218,7 @@ class VisualizationsView(views.AuthorizationMixin, ListView):
return self.model.all_visualizations() return self.model.all_visualizations()
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
ctx = super(VisualizationsView, self).get_context_data(**kwargs) ctx = super().get_context_data(**kwargs)
ctx['request'] = self.request ctx['request'] = self.request
return ctx return ctx
@ -251,12 +251,14 @@ class VisualizationsJSONView(MultipleObjectMixin, View):
sig = hashlib.sha1(force_bytes(sig)).hexdigest() sig = hashlib.sha1(force_bytes(sig)).hexdigest()
path += '?signature=' + sig path += '?signature=' + sig
data_uri = reverse('visualization-json', kwargs={'pk': visualization.pk}) data_uri = reverse('visualization-json', kwargs={'pk': visualization.pk})
data.append({ data.append(
'name': visualization.name, {
'slug': visualization.slug, 'name': visualization.name,
'path': request.build_absolute_uri(path), 'slug': visualization.slug,
'data-url': request.build_absolute_uri(data_uri), 'path': request.build_absolute_uri(path),
}) 'data-url': request.build_absolute_uri(data_uri),
}
)
response = HttpResponse(content_type='application/json') response = HttpResponse(content_type='application/json')
response.write(json.dumps(data)) response.write(json.dumps(data))
return response return response
@ -294,14 +296,16 @@ class VisualizationGeoJSONView(generics.GenericAPIView):
for cell in row.dimensions: for cell in row.dimensions:
properties[cell.dimension.label] = '%s' % (cell,) properties[cell.dimension.label] = '%s' % (cell,)
points = row.measures[0].value or [] points = row.measures[0].value or []
geojson['features'].append({ geojson['features'].append(
'type': 'Feature', {
'geometry': { 'type': 'Feature',
'type': 'MultiPoint', 'geometry': {
'coordinates': [[coord for coord in point] for point in points], 'type': 'MultiPoint',
}, 'coordinates': [[coord for coord in point] for point in points],
'properties': properties, },
}) 'properties': properties,
}
)
return Response(geojson) return Response(geojson)
@ -343,29 +347,30 @@ class VisualizationJSONView(generics.GenericAPIView):
elif len(drilldowns) == 0: elif len(drilldowns) == 0:
data = cell_value(visualization.data()[0].measures[0]) data = cell_value(visualization.data()[0].measures[0])
axis = {} axis = {}
loop.append({ loop.append({'data': data, 'axis': axis})
'data': data,
'axis': axis
})
if not all_visualizations.loop: if not all_visualizations.loop:
data = loop[0]['data'] data = loop[0]['data']
axis = loop[0]['axis'] axis = loop[0]['axis']
else: else:
axis = loop[0]['axis'] axis = loop[0]['axis']
axis['loop'] = [x.label for x in all_visualizations.loop.members(all_visualizations.filters.items())] axis['loop'] = [
x.label for x in all_visualizations.loop.members(all_visualizations.filters.items())
]
data = [x['data'] for x in loop] data = [x['data'] for x in loop]
unit = 'seconds' if all_visualizations.measure.type == 'duration' else None unit = 'seconds' if all_visualizations.measure.type == 'duration' else None
measure = all_visualizations.measure.type measure = all_visualizations.measure.type
return Response({ return Response(
'data': data, {
'axis': axis, 'data': data,
'format': '1', 'axis': axis,
'unit': unit, # legacy, prefer measure. 'format': '1',
'measure': measure, 'unit': unit, # legacy, prefer measure.
}) 'measure': measure,
}
)
class ExportVisualizationView(views.AuthorizationMixin, DetailView): class ExportVisualizationView(views.AuthorizationMixin, DetailView):
@ -387,8 +392,7 @@ class VisualizationsImportView(views.AuthorizationMixin, FormView):
def form_valid(self, form): def form_valid(self, form):
try: try:
visualizations_json = json.loads( visualizations_json = json.loads(force_text(self.request.FILES['visualizations_json'].read()))
force_text(self.request.FILES['visualizations_json'].read()))
except ValueError: except ValueError:
form.add_error('visualizations_json', _('File is not in the expected JSON format.')) form.add_error('visualizations_json', _('File is not in the expected JSON format.'))
return self.form_invalid(form) return self.form_invalid(form)
@ -401,24 +405,31 @@ class VisualizationsImportView(views.AuthorizationMixin, FormView):
if results.get('created') == 0: if results.get('created') == 0:
message1 = _('No visualization created.') message1 = _('No visualization created.')
else: else:
message1 = ungettext( message1 = (
'A visualization has been created.', ungettext(
'%(count)d visualizations have been created.', 'A visualization has been created.',
results['created']) % {'count': results['created']} '%(count)d visualizations have been created.',
results['created'],
)
% {'count': results['created']}
)
if results.get('updated') == 0: if results.get('updated') == 0:
message2 = _('No visualization updated.') message2 = _('No visualization updated.')
else: else:
message2 = ungettext( message2 = (
'A visualization has been updated.', ungettext(
'%(count)d visualizations have been updated.', 'A visualization has been updated.',
results['updated']) % {'count': results['updated']} '%(count)d visualizations have been updated.',
messages.info(self.request, u'%s %s' % (message1, message2)) results['updated'],
)
% {'count': results['updated']}
)
messages.info(self.request, '%s %s' % (message1, message2))
return super(VisualizationsImportView, self).form_valid(form) return super().form_valid(form)
class VisualizationsExportView(views.AuthorizationMixin, View): class VisualizationsExportView(views.AuthorizationMixin, View):
def get(self, request, *args, **kwargs): def get(self, request, *args, **kwargs):
response = HttpResponse(content_type='application/json') response = HttpResponse(content_type='application/json')
response['Content-Disposition'] = ( response['Content-Disposition'] = (
@ -429,12 +440,12 @@ class VisualizationsExportView(views.AuthorizationMixin, View):
class Select2ChoicesView(View): class Select2ChoicesView(View):
def get(self, request, *args, **kwargs): def get(self, request, *args, **kwargs):
widget = self.get_widget_or_404() widget = self.get_widget_or_404()
try: try:
warehouse = Engine([warehouse for warehouse in get_warehouses() warehouse = Engine(
if warehouse.name == widget.warehouse][0]) [warehouse for warehouse in get_warehouses() if warehouse.name == widget.warehouse][0]
)
cube = warehouse[widget.cube] cube = warehouse[widget.cube]
self.dimension = cube.dimensions[widget.dimension] self.dimension = cube.dimensions[widget.dimension]
except IndexError: except IndexError:
@ -448,10 +459,12 @@ class Select2ChoicesView(View):
term = request.GET.get('term', '') term = request.GET.get('term', '')
choices = self.get_choices(term, page_number, widget.max_results) choices = self.get_choices(term, page_number, widget.max_results)
return JsonResponse({ return JsonResponse(
'results': [{'text': label, 'id': s} for s, label in choices], {
'more': not(len(choices) < widget.max_results), 'results': [{'text': label, 'id': s} for s, label in choices],
}) 'more': not (len(choices) < widget.max_results),
}
)
def get_choices(self, term, page_number, max_results): def get_choices(self, term, page_number, max_results):
members = [] members = []
@ -460,7 +473,7 @@ class Select2ChoicesView(View):
members.append((None, '__none__', _('None'))) members.append((None, '__none__', _('None')))
choices = [(s, label) for v, s, label in members if term in label.lower()] choices = [(s, label) for v, s, label in members if term in label.lower()]
choices = choices[page_number * max_results:(page_number * max_results) + max_results] choices = choices[page_number * max_results : (page_number * max_results) + max_results]
return choices return choices
def get_widget_or_404(self): def get_widget_or_404(self):

4
debian/debian_config.py vendored
View File

@ -13,13 +13,13 @@ exec(open('/usr/lib/hobo/debian_config_common.py').read())
# SAML2 authentication # SAML2 authentication
AUTHENTICATION_BACKENDS = ('mellon.backends.SAMLBackend',) AUTHENTICATION_BACKENDS = ('mellon.backends.SAMLBackend',)
MELLON_ATTRIBUTE_MAPPING = { MELLON_ATTRIBUTE_MAPPING = {
'email': '{attributes[email][0]}', 'email': '{attributes[email][0]}',
'first_name': '{attributes[first_name][0]}', 'first_name': '{attributes[first_name][0]}',
'last_name': '{attributes[last_name][0]}', 'last_name': '{attributes[last_name][0]}',
} }
MELLON_SUPERUSER_MAPPING = { MELLON_SUPERUSER_MAPPING = {
'is_superuser': 'true', 'is_superuser': 'true',
} }

1
debian/scripts/warehouse_slug.py vendored
View File

@ -10,7 +10,6 @@ sudo -u bijoe bijoe-manage tenant_command runscript --all-tenants /usr/share/doc
from bijoe.utils import get_warehouses from bijoe.utils import get_warehouses
from bijoe.visualization.models import Visualization from bijoe.visualization.models import Visualization
warehouses = get_warehouses() warehouses = get_warehouses()
for visu in Visualization.objects.all(): for visu in Visualization.objects.all():
for warehouse in warehouses: for warehouse in warehouses:

6
debian/settings.py vendored
View File

@ -14,15 +14,15 @@
# SECURITY WARNING: don't run with debug turned on in production! # SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False DEBUG = False
#ADMINS = ( # ADMINS = (
# # ('User 1', 'watchdog@example.net'), # # ('User 1', 'watchdog@example.net'),
# # ('User 2', 'janitor@example.net'), # # ('User 2', 'janitor@example.net'),
#) # )
# ALLOWED_HOSTS must be correct in production! # ALLOWED_HOSTS must be correct in production!
# See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts # See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
ALLOWED_HOSTS = [ ALLOWED_HOSTS = [
'*', '*',
] ]
# Databases # Databases

2
format_json_fixtures.py
View File

@ -1,5 +1,3 @@
from __future__ import print_function
import json import json
import os import os

86
setup.py
View File

@ -1,14 +1,15 @@
#! /usr/bin/env python #! /usr/bin/env python
import sys
import subprocess
import os import os
import subprocess
import sys
from distutils.cmd import Command from distutils.cmd import Command
from distutils.command.build import build as _build from distutils.command.build import build as _build
from setuptools import setup, find_packages
from setuptools.command.sdist import sdist from setuptools import find_packages, setup
from setuptools.command.install_lib import install_lib as _install_lib from setuptools.command.install_lib import install_lib as _install_lib
from setuptools.command.sdist import sdist
class eo_sdist(sdist): class eo_sdist(sdist):
def run(self): def run(self):
@ -24,21 +25,21 @@ class eo_sdist(sdist):
def get_version(): def get_version():
'''Use the VERSION, if absent generates a version with git describe, if not """Use the VERSION, if absent generates a version with git describe, if not
tag exists, take 0.0.0- and add the length of the commit log. tag exists, take 0.0.0- and add the length of the commit log.
''' """
if os.path.exists('VERSION'): if os.path.exists('VERSION'):
with open('VERSION', 'r') as v: with open('VERSION') as v:
return v.read() return v.read()
if os.path.exists('.git'): if os.path.exists('.git'):
p = subprocess.Popen(['git', 'describe', '--dirty', '--match=v*'], stdout=subprocess.PIPE, p = subprocess.Popen(
stderr=subprocess.PIPE) ['git', 'describe', '--dirty', '--match=v*'], stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
result = p.communicate()[0] result = p.communicate()[0]
if p.returncode == 0: if p.returncode == 0:
result = result.decode('ascii').split()[0][1:] result = result.decode('ascii').split()[0][1:]
else: else:
result = '0.0.0-%s' % len(subprocess.check_output( result = '0.0.0-%s' % len(subprocess.check_output(['git', 'rev-list', 'HEAD']).splitlines())
['git', 'rev-list', 'HEAD']).splitlines())
return result.replace('-', '.').replace('.g', '+g') return result.replace('-', '.').replace('.g', '+g')
return '0.0.0' return '0.0.0'
@ -56,6 +57,7 @@ class compile_translations(Command):
def run(self): def run(self):
try: try:
from django.core.management import call_command from django.core.management import call_command
for path, dirs, files in os.walk('bijoe'): for path, dirs, files in os.walk('bijoe'):
if 'locale' not in dirs: if 'locale' not in dirs:
continue continue
@ -77,27 +79,37 @@ class install_lib(_install_lib):
_install_lib.run(self) _install_lib.run(self)
setup(name="bijoe", setup(
version=get_version(), name="bijoe",
license="AGPLv3+", version=get_version(),
description="BI daashboard from PostgreSQL start schema", license="AGPLv3+",
long_description=open('README.rst').read(), description="BI daashboard from PostgreSQL start schema",
url="http://dev.entrouvert.org/projects/publik-bi/", long_description=open('README.rst').read(),
author="Entr'ouvert", url="http://dev.entrouvert.org/projects/publik-bi/",
author_email="authentic@listes.entrouvert.com", author="Entr'ouvert",
maintainer="Benjamin Dauvergne", author_email="authentic@listes.entrouvert.com",
maintainer_email="bdauvergne@entrouvert.com", maintainer="Benjamin Dauvergne",
packages=find_packages(), maintainer_email="bdauvergne@entrouvert.com",
include_package_data=True, packages=find_packages(),
install_requires=['requests', 'django>=1.11, <2.3', 'psycopg2', 'isodate', 'Django-Select2<6', include_package_data=True,
'XStatic-ChartNew.js', 'gadjo', 'django-jsonfield<1.3', install_requires=[
'python-dateutil', 'requests',
'djangorestframework', 'django>=1.11, <2.3',
'xstatic-select2'], 'psycopg2',
scripts=['manage.py'], 'isodate',
cmdclass={ 'Django-Select2<6',
'sdist': eo_sdist, 'XStatic-ChartNew.js',
'build': build, 'gadjo',
'install_lib': install_lib, 'django-jsonfield<1.3',
'compile_translations': compile_translations, 'python-dateutil',
}) 'djangorestframework',
'xstatic-select2',
],
scripts=['manage.py'],
cmdclass={
'sdist': eo_sdist,
'build': build,
'install_lib': install_lib,
'compile_translations': compile_translations,
},
)

41
tests/conftest.py
View File

@ -1,26 +1,25 @@
import os
import glob import glob
import json import json
from contextlib import closing, contextmanager import os
import shutil
import subprocess import subprocess
import tempfile import tempfile
import shutil from contextlib import closing, contextmanager
import pytest
import django_webtest import django_webtest
import psycopg2 import psycopg2
import pytest
from django.db import transaction
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.core.management import call_command from django.core.management import call_command
from django.db import transaction
def pytest_addoption(parser): def pytest_addoption(parser):
parser.addoption( parser.addoption(
'--bijoe-store-table', action='store_true', default=False, help='Store tables value in new_tables.json', '--bijoe-store-table',
action='store_true',
default=False,
help='Store tables value in new_tables.json',
) )
@ -44,14 +43,14 @@ def john_doe(db):
@pytest.fixture @pytest.fixture
def admin(db): def admin(db):
u = User(username='super.user', first_name='Super', last_name='User', u = User(username='super.user', first_name='Super', last_name='User', email='super.user@example.net')
email='super.user@example.net')
u.set_password('super.user') u.set_password('super.user')
u.is_superuser = True u.is_superuser = True
u.is_staff = True u.is_staff = True
u.save() u.save()
return u return u
SCHEMA_PATHS = os.path.join(os.path.dirname(__file__), 'fixtures/') SCHEMA_PATHS = os.path.join(os.path.dirname(__file__), 'fixtures/')
@ -83,9 +82,19 @@ def load_schema_db(schema):
# load data # load data
for sql_path in sorted(glob.glob(os.path.join(schema_dir, '*.sql'))): for sql_path in sorted(glob.glob(os.path.join(schema_dir, '*.sql'))):
process = subprocess.Popen(['psql', '-c', '\\set ON_ERROR_STOP on', '--single-transaction', database_name, '-f', sql_path], process = subprocess.Popen(
stdout=subprocess.PIPE, [
stderr=subprocess.PIPE) 'psql',
'-c',
'\\set ON_ERROR_STOP on',
'--single-transaction',
database_name,
'-f',
sql_path,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate() stdout, stderr = process.communicate()
return_code = process.returncode return_code = process.returncode
assert return_code == 0, [stdout, stderr] assert return_code == 0, [stdout, stderr]
@ -97,7 +106,7 @@ def load_schema_db(schema):
'schema_dir': schema_dir, 'schema_dir': schema_dir,
'database_name': database_name, 'database_name': database_name,
'bijoe_schemas': [os.path.join(bijoe_schema_dir, '*_schema.json')], 'bijoe_schemas': [os.path.join(bijoe_schema_dir, '*_schema.json')],
'fixtures': fixtures 'fixtures': fixtures,
} }
tables_path = os.path.join(schema_dir, 'tables.json') tables_path = os.path.join(schema_dir, 'tables.json')
if os.path.exists(tables_path): if os.path.exists(tables_path):

128
tests/test_bijoe_schemas.py
View File

@ -1,65 +1,71 @@
# -*- coding: utf-8 -*-
from bijoe.schemas import Warehouse from bijoe.schemas import Warehouse
def test_simple_parsing(): def test_simple_parsing():
Warehouse.from_json({ Warehouse.from_json(
'name': 'coin', {
'label': 'coin', 'name': 'coin',
'pg_dsn': 'dbname=zozo', 'label': 'coin',
'search_path': ['cam', 'public'], 'pg_dsn': 'dbname=zozo',
'cubes': [ 'search_path': ['cam', 'public'],
{ 'cubes': [
'name': 'all_formdata', {
'label': 'Tous les formulaires', 'name': 'all_formdata',
'fact_table': 'formdata', 'label': 'Tous les formulaires',
'key': 'id', 'fact_table': 'formdata',
'joins': [ 'key': 'id',
{ 'joins': [
'name': 'formdef', {
'master': '{fact_table}.formdef_id', 'name': 'formdef',
'table': 'formdef', 'master': '{fact_table}.formdef_id',
'detail': 'formdef.id', 'table': 'formdef',
} 'detail': 'formdef.id',
], }
'dimensions': [ ],
{ 'dimensions': [
'label': 'formulaire', {
'name': 'formdef', 'label': 'formulaire',
'type': 'integer', 'name': 'formdef',
'join': ['formdef'], 'type': 'integer',
'value': 'formdef.id', 'join': ['formdef'],
'value_label': 'formdef.label', 'value': 'formdef.id',
'order_by': 'formdef.label' 'value_label': 'formdef.label',
}, 'order_by': 'formdef.label',
{ },
'name': 'receipt_time', {
'label': 'date de soumission', 'name': 'receipt_time',
'join': ['receipt_time'], 'label': 'date de soumission',
'type': 'date', 'join': ['receipt_time'],
'value': 'receipt_time.date' 'type': 'date',
} 'value': 'receipt_time.date',
], },
'measures': [ ],
{ 'measures': [
'type': 'integer', {
'label': 'Nombre de demandes', 'type': 'integer',
'expression': 'count({fact_table}.id)', 'label': 'Nombre de demandes',
'name': 'count' 'expression': 'count({fact_table}.id)',
}, 'name': 'count',
{ },
'type': 'integer', {
'label': u'Délai de traitement', 'type': 'integer',
'expression': 'avg((to_char(endpoint_delay, \'9999.999\') || \' days\')::interval)', 'label': 'Délai de traitement',
'name': 'avg_endpoint_delay' 'expression': (
}, 'avg((to_char(endpoint_delay, \'9999.999\') || \' days\')::interval)'
{ ),
'type': 'percent', 'name': 'avg_endpoint_delay',
'label': 'Pourcentage', },
'expression': 'count({fact_table}.id) * 100. / (select count({fact_table}.id) from {table_expression} where {where_conditions})', {
'name': 'percentage' 'type': 'percent',
} 'label': 'Pourcentage',
] 'expression': (
} 'count({fact_table}.id) * 100. / (select count({fact_table}.id) from'
], ' {table_expression} where {where_conditions})'
}) ),
'name': 'percentage',
},
],
}
],
}
)

8
tests/test_hobo_agent.py
View File

@ -20,9 +20,13 @@ from bijoe.hobo_agent.management.commands import hobo_deploy
def test_schema_from_url(): def test_schema_from_url():
for hash_length in [4, 5, 6, 7]: for hash_length in [4, 5, 6, 7]:
for length in [64, 65, 66]: for length in [64, 65, 66]:
assert len(hobo_deploy.schema_from_url('https://' + ('x' * length), hash_length=hash_length)) == 63 assert (
len(hobo_deploy.schema_from_url('https://' + ('x' * length), hash_length=hash_length)) == 63
)
schema = hobo_deploy.schema_from_url('https://demarches-saint-didier-au-mont-dor.guichet-recette.grandlyon.com/') schema = hobo_deploy.schema_from_url(
'https://demarches-saint-didier-au-mont-dor.guichet-recette.grandlyon.com/'
)
assert len(schema) == 63 assert len(schema) == 63
assert schema == 'demarches_saint_didier_au_mo0757cfguichet_recette_grandlyon_com' assert schema == 'demarches_saint_didier_au_mo0757cfguichet_recette_grandlyon_com'

14
tests/test_hobo_deploy.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# bijoe - BI dashboard # bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert # Copyright (C) 2015 Entr'ouvert
# #
@ -17,13 +16,10 @@
from contextlib import contextmanager from contextlib import contextmanager
from psycopg2.extensions import parse_dsn
import pytest import pytest
from django.utils.six.moves import configparser as ConfigParser
import sentry_sdk import sentry_sdk
from django.utils.six.moves import configparser as ConfigParser
from psycopg2.extensions import parse_dsn
from bijoe.hobo_agent.management.commands import hobo_deploy from bijoe.hobo_agent.management.commands import hobo_deploy
@ -33,7 +29,7 @@ def donothing(tenant):
yield yield
class FakeTenant(object): class FakeTenant:
domain_url = 'fake.tenant.com' domain_url = 'fake.tenant.com'
def __init__(self, directory): def __init__(self, directory):
@ -45,9 +41,7 @@ class FakeTenant(object):
@pytest.fixture @pytest.fixture
def sentry(): def sentry():
sentry_sdk.init( sentry_sdk.init(dsn='https://1234@sentry.example.com/1', environment='prod')
dsn='https://1234@sentry.example.com/1',
environment='prod')
yield yield
sentry_sdk.init() sentry_sdk.init()

8
tests/test_import_export.py
View File

@ -1,7 +1,3 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json import json
import os import os
import shutil import shutil
@ -13,8 +9,8 @@ from django.core.management import call_command
from django.utils.encoding import force_bytes from django.utils.encoding import force_bytes
from django.utils.six import StringIO from django.utils.six import StringIO
from bijoe.visualization.models import Visualization
from bijoe.utils import import_site from bijoe.utils import import_site
from bijoe.visualization.models import Visualization
pytestmark = pytest.mark.django_db pytestmark = pytest.mark.django_db
@ -35,7 +31,7 @@ def test_import_export(schema1, app):
'representation': 'table', 'representation': 'table',
'loop': '', 'loop': '',
'filters': {}, 'filters': {},
'drilldown_x': 'date__yearmonth' 'drilldown_x': 'date__yearmonth',
} }
def create_visu(i=0): def create_visu(i=0):

20
tests/test_relative_time.py
View File

@ -1,16 +1,16 @@
# -*- coding: utf-8 -*-
from datetime import date from datetime import date
from bijoe.relative_time import RelativeDate from bijoe.relative_time import RelativeDate
def test_relative_date(): def test_relative_date():
today = date(2016, 3, 3) today = date(2016, 3, 3)
assert RelativeDate(u'cette année', today=today) == date(2016, 1, 1) assert RelativeDate('cette année', today=today) == date(2016, 1, 1)
assert RelativeDate(u'ce mois', today=today) == date(2016, 3, 1) assert RelativeDate('ce mois', today=today) == date(2016, 3, 1)
assert RelativeDate(u'le mois dernier', today=today) == date(2016, 2, 1) assert RelativeDate('le mois dernier', today=today) == date(2016, 2, 1)
assert RelativeDate(u'les 4 derniers mois', today=today) == date(2015, 11, 1) assert RelativeDate('les 4 derniers mois', today=today) == date(2015, 11, 1)
assert RelativeDate(u'le mois prochain', today=today) == date(2016, 4, 1) assert RelativeDate('le mois prochain', today=today) == date(2016, 4, 1)
assert RelativeDate(u'les 3 prochains mois', today=today) == date(2016, 6, 1) assert RelativeDate('les 3 prochains mois', today=today) == date(2016, 6, 1)
assert RelativeDate(u' cette semaine', today=today) == date(2016, 2, 29) assert RelativeDate(' cette semaine', today=today) == date(2016, 2, 29)
assert RelativeDate(u' maintenant', today=today) == today assert RelativeDate(' maintenant', today=today) == today
assert RelativeDate(u'2016-01-01', today=today) == date(2016, 1, 1) assert RelativeDate('2016-01-01', today=today) == date(2016, 1, 1)

309
tests/test_schema1.py
View File

@ -1,11 +1,9 @@
# -*- coding: utf-8 -*-
import json import json
from utils import login, get_table, get_ods_table, get_ods_document, request_select2 from utils import get_ods_document, get_ods_table, get_table, login, request_select2
from bijoe.visualization.ods import OFFICE_NS, TABLE_NS
from bijoe.visualization.models import Visualization as VisualizationModel from bijoe.visualization.models import Visualization as VisualizationModel
from bijoe.visualization.ods import OFFICE_NS, TABLE_NS
from bijoe.visualization.utils import Visualization from bijoe.visualization.utils import Visualization
@ -15,7 +13,7 @@ def test_simple(schema1, app, admin):
response = response.click('schema1') response = response.click('schema1')
response = response.click('Facts 1') response = response.click('Facts 1')
assert 'big-msg-info' in response assert 'big-msg-info' in response
assert u'le champ « pouët »' in response assert 'le champ « pouët »' in response
assert 'warning2' in response assert 'warning2' in response
form = response.form form = response.form
form.set('representation', 'table') form.set('representation', 'table')
@ -24,9 +22,21 @@ def test_simple(schema1, app, admin):
response = form.submit('visualize') response = form.submit('visualize')
assert 'big-msg-info' not in response assert 'big-msg-info' not in response
assert get_table(response) == [ assert get_table(response) == [
['Inner SubCategory', u'sub\xe910', u'sub\xe911', u'sub\xe94', u'sub\xe95', u'sub\xe96', u'sub\xe98', [
u'sub\xe99', u'sub\xe97', u'sub\xe92', u'sub\xe93', u'sub\xe91'], 'Inner SubCategory',
['number of rows', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '15'] 'sub\xe910',
'sub\xe911',
'sub\xe94',
'sub\xe95',
'sub\xe96',
'sub\xe98',
'sub\xe99',
'sub\xe97',
'sub\xe92',
'sub\xe93',
'sub\xe91',
],
['number of rows', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '15'],
] ]
form = response.form form = response.form
form.set('representation', 'table') form.set('representation', 'table')
@ -35,7 +45,7 @@ def test_simple(schema1, app, admin):
response = form.submit('visualize') response = form.submit('visualize')
assert 'big-msg-info' not in response assert 'big-msg-info' not in response
assert get_table(response) == [ assert get_table(response) == [
['mois (Date)', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt'], ['mois (Date)', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt'],
['number of rows', '10', '1', '1', '1', '1', '1', '1', '1'], ['number of rows', '10', '1', '1', '1', '1', '1', '1', '1'],
] ]
@ -54,14 +64,14 @@ def test_truncated_previous_year_range(schema1, app, admin, freezer):
freezer.move_to('2019-01-01 01:00:00') freezer.move_to('2019-01-01 01:00:00')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [
['', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt', 'Total'], ['', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt', 'Total'],
['2017', '0', '0', '0', '0', '0', '0', '0', '0', '0'], ['2017', '0', '0', '0', '0', '0', '0', '0', '0', '0'],
] ]
freezer.move_to('2018-01-01 01:00:00') freezer.move_to('2018-01-01 01:00:00')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [
['', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt', 'Total'], ['', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt', 'Total'],
['2017', '10', '1', '1', '1', '1', '1', '1', '1', '17'], ['2017', '10', '1', '1', '1', '1', '1', '1', '1', '17'],
] ]
@ -77,7 +87,9 @@ def test_boolean_dimension(schema1, app, admin):
form.set('drilldown_x', 'boolean') form.set('drilldown_x', 'boolean')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [['Boolean', 'Oui', 'Non'], ['number of rows', '8', '9']] assert get_table(response) == [['Boolean', 'Oui', 'Non'], ['number of rows', '8', '9']]
form['filter__boolean'].force_value([o[0] for o in form.fields['filter__boolean'][0].options if o[2] == 'Oui'][0]) form['filter__boolean'].force_value(
[o[0] for o in form.fields['filter__boolean'][0].options if o[2] == 'Oui'][0]
)
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [['Boolean', 'Oui', 'Non'], ['number of rows', '8', '0']] assert get_table(response) == [['Boolean', 'Oui', 'Non'], ['number of rows', '8', '0']]
@ -92,7 +104,10 @@ def test_string_dimension(schema1, app, admin):
form.set('measure', 'simple_count') form.set('measure', 'simple_count')
form.set('drilldown_x', 'string') form.set('drilldown_x', 'string')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [['String', 'a', 'b', 'c', 'Aucun(e)'], ['number of rows', '11', '2', '3', '1']] assert get_table(response) == [
['String', 'a', 'b', 'c', 'Aucun(e)'],
['number of rows', '11', '2', '3', '1'],
]
form['filter__string'].force_value(['a', 'b', '__none__']) form['filter__string'].force_value(['a', 'b', '__none__'])
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [['String', 'a', 'b', 'Aucun(e)'], ['number of rows', '11', '2', '1']] assert get_table(response) == [['String', 'a', 'b', 'Aucun(e)'], ['number of rows', '11', '2', '1']]
@ -100,18 +115,20 @@ def test_string_dimension(schema1, app, admin):
def test_string_dimension_json_data(schema1, app, admin): def test_string_dimension_json_data(schema1, app, admin):
# test conversion to Javascript declaration # test conversion to Javascript declaration
visu = Visualization.from_json({ visu = Visualization.from_json(
'warehouse': 'schema1', {
'cube': 'facts1', 'warehouse': 'schema1',
'representation': 'table', 'cube': 'facts1',
'measure': 'simple_count', 'representation': 'table',
'drilldown_x': 'string' 'measure': 'simple_count',
}) 'drilldown_x': 'string',
}
)
assert json.loads(json.dumps(visu.json_data())) == [ assert json.loads(json.dumps(visu.json_data())) == [
{u'coords': [{u'value': u'a'}], u'measures': [{u'value': 11}]}, {'coords': [{'value': 'a'}], 'measures': [{'value': 11}]},
{u'coords': [{u'value': u'b'}], u'measures': [{u'value': 2}]}, {'coords': [{'value': 'b'}], 'measures': [{'value': 2}]},
{u'coords': [{u'value': u'c'}], u'measures': [{u'value': 3}]}, {'coords': [{'value': 'c'}], 'measures': [{'value': 3}]},
{u'coords': [{u'value': u'Aucun(e)'}], u'measures': [{u'value': 1}]} {'coords': [{'value': 'Aucun(e)'}], 'measures': [{'value': 1}]},
] ]
@ -126,16 +143,26 @@ def test_item_dimension(schema1, app, admin):
form.set('drilldown_x', 'outersubcategory') form.set('drilldown_x', 'outersubcategory')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [
['Outer SubCategory', u'sub\xe910', u'sub\xe911', u'sub\xe94', u'sub\xe95', u'sub\xe96', u'sub\xe98', [
u'sub\xe99', u'sub\xe97', u'sub\xe92', u'sub\xe93', u'sub\xe91', 'Aucun(e)'], 'Outer SubCategory',
['number of rows', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '15', '1'] 'sub\xe910',
'sub\xe911',
'sub\xe94',
'sub\xe95',
'sub\xe96',
'sub\xe98',
'sub\xe99',
'sub\xe97',
'sub\xe92',
'sub\xe93',
'sub\xe91',
'Aucun(e)',
],
['number of rows', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '15', '1'],
] ]
form['filter__outersubcategory'].force_value(['__none__']) form['filter__outersubcategory'].force_value(['__none__'])
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [['Outer SubCategory', 'Aucun(e)'], ['number of rows', '1']]
['Outer SubCategory', 'Aucun(e)'],
['number of rows', '1']
]
def test_yearmonth_drilldown(schema1, app, admin): def test_yearmonth_drilldown(schema1, app, admin):
@ -149,9 +176,18 @@ def test_yearmonth_drilldown(schema1, app, admin):
form.set('drilldown_x', 'date__yearmonth') form.set('drilldown_x', 'date__yearmonth')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [
[u'ann\xe9e et mois (Date)', '01/2017', '02/2017', '03/2017', [
'04/2017', '05/2017', '06/2017', '07/2017', '08/2017'], 'ann\xe9e et mois (Date)',
['number of rows', '10', '1', '1', '1', '1', '1', '1', '1'] '01/2017',
'02/2017',
'03/2017',
'04/2017',
'05/2017',
'06/2017',
'07/2017',
'08/2017',
],
['number of rows', '10', '1', '1', '1', '1', '1', '1', '1'],
] ]
@ -196,53 +232,47 @@ def test_truncated_previous_year_range_on_datetime(schema1, app, admin, freezer)
freezer.move_to('2019-01-01 01:00:00') freezer.move_to('2019-01-01 01:00:00')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [
['', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt', 'Total'], ['', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt', 'Total'],
['2017', '0', '0', '0', '0', '0', '0', '0', '0', '0'], ['2017', '0', '0', '0', '0', '0', '0', '0', '0', '0'],
] ]
freezer.move_to('2018-01-01 01:00:00') freezer.move_to('2018-01-01 01:00:00')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [
['', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt', 'Total'], ['', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt', 'Total'],
['2017', '10', '1', '1', '1', '1', '1', '1', '1', '17'], ['2017', '10', '1', '1', '1', '1', '1', '1', '1', '17'],
] ]
def test_none_percent_json_data_0d(schema1, app, admin): def test_none_percent_json_data_0d(schema1, app, admin):
# test conversion to Javascript declaration # test conversion to Javascript declaration
visu = Visualization.from_json({ visu = Visualization.from_json(
'warehouse': 'schema1', {
'cube': 'facts1', 'warehouse': 'schema1',
'representation': 'graphical', 'cube': 'facts1',
'measure': 'percent', 'representation': 'graphical',
}) 'measure': 'percent',
assert visu.json_data() == [{u'coords': [], u'measures': [{u'value': 100.0}]}] }
)
assert visu.json_data() == [{'coords': [], 'measures': [{'value': 100.0}]}]
def test_none_percent_json_data_2d(schema1, app, admin): def test_none_percent_json_data_2d(schema1, app, admin):
# test conversion to Javascript declaration # test conversion to Javascript declaration
visu = Visualization.from_json({ visu = Visualization.from_json(
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'percent',
'drilldown_y': 'leftcategory',
'drilldown_x': 'date__year',
})
assert visu.json_data() == [
{ {
'coords': [{'value': u'2017'}, {'value': u'cat\xe92'}], 'warehouse': 'schema1',
'measures': [{'value': 0}] 'cube': 'facts1',
}, 'representation': 'graphical',
{ 'measure': 'percent',
'coords': [{'value': u'2017'}, {'value': u'cat\xe93'}], 'drilldown_y': 'leftcategory',
'measures': [{'value': 0}]}, 'drilldown_x': 'date__year',
{
'coords': [{'value': u'2017'}, {'value': u'cat\xe91'}],
'measures': [{'value': 94.11764705882354}]},
{
'coords': [{'value': u'2017'}, {'value': u'Aucun(e)'}],
'measures': [{'value': 5.882352941176471}]
} }
)
assert visu.json_data() == [
{'coords': [{'value': '2017'}, {'value': 'cat\xe92'}], 'measures': [{'value': 0}]},
{'coords': [{'value': '2017'}, {'value': 'cat\xe93'}], 'measures': [{'value': 0}]},
{'coords': [{'value': '2017'}, {'value': 'cat\xe91'}], 'measures': [{'value': 94.11764705882354}]},
{'coords': [{'value': '2017'}, {'value': 'Aucun(e)'}], 'measures': [{'value': 5.882352941176471}]},
] ]
@ -258,96 +288,89 @@ def test_geoloc(schema1, app, admin):
'measure': 'percent', 'measure': 'percent',
'drilldown_y': 'outercategory', 'drilldown_y': 'outercategory',
'drilldown_x': 'date__year', 'drilldown_x': 'date__year',
}) },
)
response = app.get('/visualization/%d/geojson/' % visu.pk) response = app.get('/visualization/%d/geojson/' % visu.pk)
assert response.json == { assert response.json == {
'type': 'FeatureCollection', 'type': 'FeatureCollection',
'features': [{ 'features': [
u'geometry': { {
u'coordinates': [ 'geometry': {
[1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], 'coordinates': [
[1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0],
[1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0] [1.0, 1.0],
], [1.0, 1.0],
u'type': u'MultiPoint' [1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
],
'type': 'MultiPoint',
},
'properties': {'Outer Category': 'cat\xe91', 'ann\xe9e (Date)': '2017'},
'type': 'Feature',
}, },
u'properties': { {
u'Outer Category': u'cat\xe91', 'geometry': {'coordinates': [[1.0, 1.0]], 'type': 'MultiPoint'},
u'ann\xe9e (Date)': u'2017' 'properties': {'Outer Category': 'Aucun(e)', 'ann\xe9e (Date)': '2017'},
'type': 'Feature',
}, },
u'type': u'Feature' {
}, 'geometry': {'coordinates': [], 'type': 'MultiPoint'},
{ 'properties': {'Outer Category': 'cat\xe92', 'ann\xe9e (Date)': 'Aucun(e)'},
u'geometry': { 'type': 'Feature',
u'coordinates': [[1.0, 1.0]],
u'type': u'MultiPoint'
}, },
u'properties': { {
u'Outer Category': u'Aucun(e)', 'geometry': {'coordinates': [], 'type': 'MultiPoint'},
u'ann\xe9e (Date)': u'2017' 'properties': {'Outer Category': 'cat\xe93', 'ann\xe9e (Date)': 'Aucun(e)'},
'type': 'Feature',
}, },
u'type': u'Feature' {
}, 'geometry': {'coordinates': [], 'type': 'MultiPoint'},
{ 'properties': {'Outer Category': 'cat\xe91', 'ann\xe9e (Date)': 'Aucun(e)'},
u'geometry': { 'type': 'Feature',
u'coordinates': [],
u'type': u'MultiPoint'
}, },
u'properties': { ],
u'Outer Category': u'cat\xe92', }
u'ann\xe9e (Date)': u'Aucun(e)'
},
u'type': u'Feature'
},
{
u'geometry': {
u'coordinates': [],
u'type': u'MultiPoint'
},
u'properties': {
u'Outer Category': u'cat\xe93',
u'ann\xe9e (Date)': u'Aucun(e)'
},
u'type': u'Feature'
},
{
u'geometry': {
u'coordinates': [],
u'type': u'MultiPoint'
},
u'properties': {
u'Outer Category': u'cat\xe91',
u'ann\xe9e (Date)': u'Aucun(e)'
},
u'type': u'Feature'
}
]}
def test_filter_type_mismatch(schema1, app, admin): def test_filter_type_mismatch(schema1, app, admin):
# test conversion to Javascript declaration # test conversion to Javascript declaration
visu = Visualization.from_json({ visu = Visualization.from_json(
'warehouse': 'schema1', {
'cube': 'facts1', 'warehouse': 'schema1',
'representation': 'graphical', 'cube': 'facts1',
'measure': 'simple_count', 'representation': 'graphical',
'filters': { 'measure': 'simple_count',
'string': [1], 'filters': {
'string': [1],
},
} }
}) )
assert visu.json_data() == [{'coords': [], 'measures': [{'value': 0}]}] assert visu.json_data() == [{'coords': [], 'measures': [{'value': 0}]}]
def test_empty_filter(schema1, app, admin): def test_empty_filter(schema1, app, admin):
visu = Visualization.from_json({ visu = Visualization.from_json(
'warehouse': 'schema1', {
'cube': 'facts1', 'warehouse': 'schema1',
'representation': 'graphical', 'cube': 'facts1',
'measure': 'simple_count', 'representation': 'graphical',
'filters': { 'measure': 'simple_count',
'innercategory': [], 'filters': {
'innercategory': [],
},
} }
}) )
assert visu.json_data() == [{'coords': [], 'measures': [{'value': 17}]}] assert visu.json_data() == [{'coords': [], 'measures': [{'value': 17}]}]
@ -362,10 +385,7 @@ def test_json_dimensions(schema1, app, admin):
form.set('measure', 'simple_count') form.set('measure', 'simple_count')
form.set('drilldown_x', 'a') form.set('drilldown_x', 'a')
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [['A', 'x', 'y', 'z'], ['number of rows', '7', '9', '1']]
['A', 'x', 'y', 'z'],
['number of rows', '7', '9', '1']
]
assert 'filter__a' in form.fields assert 'filter__a' in form.fields
choices = [o['id'] for o in request_select2(app, response, 'filter__a')['results']] choices = [o['id'] for o in request_select2(app, response, 'filter__a')['results']]
@ -373,10 +393,7 @@ def test_json_dimensions(schema1, app, admin):
form['filter__a'].force_value(['x', 'y']) form['filter__a'].force_value(['x', 'y'])
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [['A', 'x', 'y', 'z'], ['number of rows', '7', '9', '0']]
['A', 'x', 'y', 'z'],
['number of rows', '7', '9', '0']
]
def test_json_dimensions_having_percent(schema1, app, admin): def test_json_dimensions_having_percent(schema1, app, admin):
@ -392,7 +409,7 @@ def test_json_dimensions_having_percent(schema1, app, admin):
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [
['A', 'x', 'y', 'z'], ['A', 'x', 'y', 'z'],
['pourcentage des demandes', '41,18 %', '52,94 %', '5,88 %'] ['pourcentage des demandes', '41,18 %', '52,94 %', '5,88 %'],
] ]
assert 'filter__a' in form.fields assert 'filter__a' in form.fields
@ -403,7 +420,7 @@ def test_json_dimensions_having_percent(schema1, app, admin):
response = form.submit('visualize') response = form.submit('visualize')
assert get_table(response) == [ assert get_table(response) == [
['A', 'x', 'y', 'z'], ['A', 'x', 'y', 'z'],
['pourcentage des demandes', '43,75 %', '56,25 %', '0,00 %'] ['pourcentage des demandes', '43,75 %', '56,25 %', '0,00 %'],
] ]

16
tests/test_schema2.py
View File

@ -4,20 +4,14 @@ import re
import pytest import pytest
from tabulate import tabulate from tabulate import tabulate
from utils import get_table, login
from utils import login, get_table
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):
if hasattr(metafunc, 'function'): if hasattr(metafunc, 'function'):
fcode = metafunc.function.__code__ fcode = metafunc.function.__code__
if 'visualization' in fcode.co_varnames[:fcode.co_argcount]: if 'visualization' in fcode.co_varnames[: fcode.co_argcount]:
with open( with open(os.path.join(os.path.dirname(__file__), 'fixtures', 'schema2', 'tables.json')) as fd:
os.path.join(
os.path.dirname(__file__),
'fixtures',
'schema2',
'tables.json')) as fd:
tables = json.load(fd) tables = json.load(fd)
metafunc.parametrize(['visualization'], [[x] for x in tables]) metafunc.parametrize(['visualization'], [[x] for x in tables])
@ -53,6 +47,4 @@ def test_simple(request, schema2, app, admin, visualization):
d[visualization] = table d[visualization] = table
with open(new_table_path, 'w') as fd: with open(new_table_path, 'w') as fd:
json.dump(d, fd, indent=4, sort_keys=True, separators=(',', ': ')) json.dump(d, fd, indent=4, sort_keys=True, separators=(',', ': '))
assert_equal_tables( assert_equal_tables(schema2['tables'][visualization], table)
schema2['tables'][visualization],
table)

16
tests/test_schemas.py
View File

@ -15,24 +15,22 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import pytest import pytest
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
from bijoe.schemas import ( from bijoe.schemas import Dimension
Dimension,
)
def test_absent_label(): def test_absent_label():
assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'integer'}).absent_label == _('None') assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'integer'}).absent_label == _('None')
assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'string'}).absent_label == _('None') assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'string'}).absent_label == _('None')
assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'bool'}).absent_label == _('N/A') assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'bool'}).absent_label == _('N/A')
assert Dimension.from_json( assert (
{'name': 'x', 'value': 'x', 'type': 'boolean', 'absent_label': 'coin'}).absent_label == 'coin' Dimension.from_json(
{'name': 'x', 'value': 'x', 'type': 'boolean', 'absent_label': 'coin'}
).absent_label
== 'coin'
)
with pytest.raises(NotImplementedError): with pytest.raises(NotImplementedError):
Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'coin'}).absent_label Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'coin'}).absent_label

25
tests/test_signature.py
View File

@ -38,7 +38,7 @@ def test_signature():
assert not signature.check_string(STRING, signature.sign_string(STRING, KEY), OTHER_KEY) assert not signature.check_string(STRING, signature.sign_string(STRING, KEY), OTHER_KEY)
assert not signature.check_query(signature.sign_query(QUERY, KEY), OTHER_KEY) assert not signature.check_query(signature.sign_query(QUERY, KEY), OTHER_KEY)
assert not signature.check_url(signature.sign_url(URL, KEY), OTHER_KEY) assert not signature.check_url(signature.sign_url(URL, KEY), OTHER_KEY)
#assert not signature.check_url('%s&foo=bar' % signature.sign_url(URL, KEY), KEY) # assert not signature.check_url('%s&foo=bar' % signature.sign_url(URL, KEY), KEY)
# Test URL is preserved # Test URL is preserved
assert URL in signature.sign_url(URL, KEY) assert URL in signature.sign_url(URL, KEY)
@ -50,28 +50,29 @@ def test_signature():
assert '&nonce=' in signature.sign_url(URL, KEY) assert '&nonce=' in signature.sign_url(URL, KEY)
# Test unicode key conversion to UTF-8 # Test unicode key conversion to UTF-8
assert signature.check_url(signature.sign_url(URL, u'\xe9\xe9'), b'\xc3\xa9\xc3\xa9') assert signature.check_url(signature.sign_url(URL, '\xe9\xe9'), b'\xc3\xa9\xc3\xa9')
assert signature.check_url(signature.sign_url(URL, b'\xc3\xa9\xc3\xa9'), u'\xe9\xe9') assert signature.check_url(signature.sign_url(URL, b'\xc3\xa9\xc3\xa9'), '\xe9\xe9')
# Test timedelta parameter # Test timedelta parameter
now = datetime.datetime.utcnow() now = datetime.datetime.utcnow()
assert '&timestamp=%s' % urllib.quote(now.strftime('%Y-%m-%dT%H:%M:%SZ')) in \ assert '&timestamp=%s' % urllib.quote(now.strftime('%Y-%m-%dT%H:%M:%SZ')) in signature.sign_url(
signature.sign_url(URL, KEY, timestamp=now) URL, KEY, timestamp=now
)
# Test nonce parameter # Test nonce parameter
assert '&nonce=uuu&' in signature.sign_url(URL, KEY, nonce='uuu') assert '&nonce=uuu&' in signature.sign_url(URL, KEY, nonce='uuu')
# Test known_nonce # Test known_nonce
assert signature.check_url(signature.sign_url(URL, KEY), KEY, assert signature.check_url(signature.sign_url(URL, KEY), KEY, known_nonce=lambda nonce: nonce == 'xxx')
known_nonce=lambda nonce: nonce == 'xxx') assert signature.check_url(
assert signature.check_url(signature.sign_url(URL, KEY, nonce='xxx'), KEY, signature.sign_url(URL, KEY, nonce='xxx'), KEY, known_nonce=lambda nonce: nonce == 'xxx'
known_nonce=lambda nonce: nonce == 'xxx') )
# Test timedelta # Test timedelta
now = (datetime.datetime.utcnow() - datetime.timedelta(seconds=29)) now = datetime.datetime.utcnow() - datetime.timedelta(seconds=29)
assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY) assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY)
now = (datetime.datetime.utcnow() - datetime.timedelta(seconds=30)) now = datetime.datetime.utcnow() - datetime.timedelta(seconds=30)
assert not signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY) assert not signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY)
now = (datetime.datetime.utcnow() - datetime.timedelta(seconds=2)) now = datetime.datetime.utcnow() - datetime.timedelta(seconds=2)
assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY) assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY)
assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY, timedelta=2) assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY, timedelta=2)

74
tests/test_views.py
View File

@ -17,19 +17,17 @@
import copy import copy
import hashlib import hashlib
import json import json
from unittest import mock
import mock
import pytest import pytest
from webtest import Upload
from django.urls import reverse from django.urls import reverse
from django.utils.encoding import force_bytes from django.utils.encoding import force_bytes
from utils import login
from webtest import Upload
from bijoe.visualization.models import Visualization from bijoe.visualization.models import Visualization
from bijoe.visualization.signature import sign_url from bijoe.visualization.signature import sign_url
from utils import login
@pytest.fixture @pytest.fixture
def visualization(): def visualization():
@ -42,7 +40,9 @@ def visualization():
'representation': 'table', 'representation': 'table',
'loop': '', 'loop': '',
'filters': {}, 'filters': {},
'drilldown_x': 'date__yearmonth'}) 'drilldown_x': 'date__yearmonth',
},
)
def test_simple_user_403(app, john_doe): def test_simple_user_403(app, john_doe):
@ -78,11 +78,13 @@ def test_visualizations_json_api(schema1, app, admin, settings):
'default': { 'default': {
'verif_orig': orig, 'verif_orig': orig,
'secret': key, 'secret': key,
}}} }
}
}
url = '%s?orig=%s' % (reverse('visualizations-json'), orig) url = '%s?orig=%s' % (reverse('visualizations-json'), orig)
url = sign_url(url, key) url = sign_url(url, key)
resp = app.get(url, status=200) resp = app.get(url, status=200)
assert set([x['slug'] for x in resp.json]) == set(['test', 'test-2', 'test-3', 'test-4']) assert {x['slug'] for x in resp.json} == {'test', 'test-2', 'test-3', 'test-4'}
url = '%s?orig=%s' % (reverse('visualizations-json'), orig) url = '%s?orig=%s' % (reverse('visualizations-json'), orig)
url = sign_url(url, 'wrong-key') url = sign_url(url, 'wrong-key')
@ -97,7 +99,7 @@ def test_visualizations_json_api(schema1, app, admin, settings):
login(app, admin) login(app, admin)
resp = app.get(reverse('visualizations-json')) resp = app.get(reverse('visualizations-json'))
assert set([x['slug'] for x in resp.json]) == set(['test', 'test-2', 'test-3', 'test-4']) assert {x['slug'] for x in resp.json} == {'test', 'test-2', 'test-3', 'test-4'}
def test_visualization_json_api(schema1, app, admin, visualization): def test_visualization_json_api(schema1, app, admin, visualization):
@ -105,7 +107,18 @@ def test_visualization_json_api(schema1, app, admin, visualization):
resp = app.get(reverse('visualization-json', kwargs={'pk': visualization.id})) resp = app.get(reverse('visualization-json', kwargs={'pk': visualization.id}))
# values from test_schem1/test_yearmonth_drilldown # values from test_schem1/test_yearmonth_drilldown
assert resp.json == { assert resp.json == {
'axis': {'x_labels': ['01/2017', '02/2017', '03/2017', '04/2017', '05/2017', '06/2017', '07/2017', '08/2017']}, 'axis': {
'x_labels': [
'01/2017',
'02/2017',
'03/2017',
'04/2017',
'05/2017',
'06/2017',
'07/2017',
'08/2017',
]
},
'data': [10, 1, 1, 1, 1, 1, 1, 1], 'data': [10, 1, 1, 1, 1, 1, 1, 1],
'format': '1', 'format': '1',
'unit': None, 'unit': None,
@ -121,9 +134,28 @@ def test_visualization_json_api_duration(schema1, app, admin, visualization):
resp = app.get(reverse('visualization-json', kwargs={'pk': visualization.id})) resp = app.get(reverse('visualization-json', kwargs={'pk': visualization.id}))
# values from test_schem1/test_yearmonth_drilldown # values from test_schem1/test_yearmonth_drilldown
assert resp.json == { assert resp.json == {
'axis': {'x_labels': ['01/2017', '02/2017', '03/2017', '04/2017', '05/2017', '06/2017', '07/2017', '08/2017']}, 'axis': {
'data': [536968800.0, 539258400.0, 541677600.0, 544352400.0, 'x_labels': [
546944400.0, 549622800.0, 552214800.0, 554893200.0], '01/2017',
'02/2017',
'03/2017',
'04/2017',
'05/2017',
'06/2017',
'07/2017',
'08/2017',
]
},
'data': [
536968800.0,
539258400.0,
541677600.0,
544352400.0,
546944400.0,
549622800.0,
552214800.0,
554893200.0,
],
'format': '1', 'format': '1',
'unit': 'seconds', 'unit': 'seconds',
'measure': 'duration', 'measure': 'duration',
@ -194,7 +226,9 @@ def test_import_visualization(schema1, app, admin, visualization, settings, free
# existing visualization # existing visualization
resp = app.get('/', status=200) resp = app.get('/', status=200)
resp = resp.click('Import') resp = resp.click('Import')
resp.form['visualizations_json'] = Upload('export.json', visualization_export.encode('utf-8'), 'application/json') resp.form['visualizations_json'] = Upload(
'export.json', visualization_export.encode('utf-8'), 'application/json'
)
resp = resp.form.submit().follow() resp = resp.form.submit().follow()
assert 'No visualization created. A visualization has been updated.' in resp.text assert 'No visualization created. A visualization has been updated.' in resp.text
assert Visualization.objects.count() == 1 assert Visualization.objects.count() == 1
@ -203,7 +237,9 @@ def test_import_visualization(schema1, app, admin, visualization, settings, free
Visualization.objects.all().delete() Visualization.objects.all().delete()
resp = app.get('/') resp = app.get('/')
resp = resp.click('Import') resp = resp.click('Import')
resp.form['visualizations_json'] = Upload('export.json', visualization_export.encode('utf-8'), 'application/json') resp.form['visualizations_json'] = Upload(
'export.json', visualization_export.encode('utf-8'), 'application/json'
)
resp = resp.form.submit().follow() resp = resp.form.submit().follow()
assert 'A visualization has been created. No visualization updated.' in resp.text assert 'A visualization has been created. No visualization updated.' in resp.text
assert Visualization.objects.count() == 1 assert Visualization.objects.count() == 1
@ -220,8 +256,8 @@ def test_import_visualization(schema1, app, admin, visualization, settings, free
resp = app.get('/', status=200) resp = app.get('/', status=200)
resp = resp.click('Import') resp = resp.click('Import')
resp.form['visualizations_json'] = Upload( resp.form['visualizations_json'] = Upload(
'export.json', 'export.json', json.dumps(visualizations).encode('utf-8'), 'application/json'
json.dumps(visualizations).encode('utf-8'), 'application/json') )
resp = resp.form.submit().follow() resp = resp.form.submit().follow()
assert '2 visualizations have been created. A visualization has been updated.' in resp.text assert '2 visualizations have been created. A visualization has been updated.' in resp.text
assert Visualization.objects.count() == 3 assert Visualization.objects.count() == 3
@ -235,7 +271,9 @@ def test_import_visualization(schema1, app, admin, visualization, settings, free
resp = app.get('/') resp = app.get('/')
resp = resp.click('Import') resp = resp.click('Import')
resp.form['visualizations_json'] = Upload('export.json', visualizations_export.encode('utf-8'), 'application/json') resp.form['visualizations_json'] = Upload(
'export.json', visualizations_export.encode('utf-8'), 'application/json'
)
resp = resp.form.submit().follow() resp = resp.form.submit().follow()
assert '3 visualizations have been created. No visualization updated.' in resp.text assert '3 visualizations have been created. No visualization updated.' in resp.text
assert Visualization.objects.count() == 3 assert Visualization.objects.count() == 3

9
tests/utils.py
View File

@ -1,6 +1,6 @@
import io import io
import zipfile
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import zipfile
from django.conf import settings from django.conf import settings
@ -35,8 +35,8 @@ def get_table(response):
def xml_node_text_content(node): def xml_node_text_content(node):
'''Extract text content from node and all its children. Equivalent to """Extract text content from node and all its children. Equivalent to
xmlNodeGetContent from libxml.''' xmlNodeGetContent from libxml."""
if node is None: if node is None:
return '' return ''
@ -50,7 +50,8 @@ def xml_node_text_content(node):
if child.tail: if child.tail:
s.append(child.tail) s.append(child.tail)
return s return s
return u''.join(helper(node))
return ''.join(helper(node))
def get_ods_document(response): def get_ods_document(response):