misc: apply pyupgrade/isort/black (#56062)

This commit is contained in:
Benjamin Dauvergne 2021-08-09 15:31:51 +02:00
parent 951bd3387f
commit 65d33f00b7
41 changed files with 922 additions and 787 deletions

View File

@ -17,14 +17,13 @@
import collections
import contextlib
import datetime
import logging
import itertools
import hashlib
import itertools
import logging
import psycopg2
from django.core.cache import cache
from django.conf import settings
from django.core.cache import cache
from django.utils.encoding import force_bytes, force_text
from django.utils.six import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
@ -63,20 +62,20 @@ class MeasureCell(collections.namedtuple('_Cell', ['measure', 'value'])):
return _('N/A')
else:
try:
return (u'%4.2f' % float(value)).replace('.', ',') + u' %'
return ('%4.2f' % float(value)).replace('.', ',') + ' %'
except TypeError:
return _('N/A')
elif self.measure.type == 'duration':
if value is None:
return u'0'
return '0'
else:
s = u''
s = ''
if value.days:
s += u'%d jour(s)' % value.days
s += '%d jour(s)' % value.days
if value.seconds // 3600:
s += u' %d heure(s)' % (value.seconds // 3600)
s += ' %d heure(s)' % (value.seconds // 3600)
if not s:
s = u'moins d\'1 heure'
s = 'moins d\'1 heure'
return s
elif self.measure.type == 'bool':
if value is None:
@ -99,7 +98,7 @@ class Cells(collections.namedtuple('Cells', ['dimensions', 'measures'])):
def __new__(cls, dimensions=[], measures=[]):
dimensions = list(dimensions)
measures = list(measures)
return super(Cells, cls).__new__(cls, dimensions, measures)
return super().__new__(cls, dimensions, measures)
def quote(s):
@ -126,7 +125,7 @@ def to_tuple(cur, values):
Member = collections.namedtuple('Member', ['id', 'label'])
class EngineDimension(object):
class EngineDimension:
def __init__(self, engine, engine_cube, dimension):
self.engine = engine
self.engine_cube = engine_cube
@ -148,9 +147,13 @@ class EngineDimension(object):
cache_key = self.cache_key(filters)
members = cache.get(cache_key)
if members is not None:
self.engine.log.debug('MEMBERS: (from cache) dimension %s.%s filters=%s: %s',
self.engine_cube.name, self.name, filters,
members)
self.engine.log.debug(
'MEMBERS: (from cache) dimension %s.%s filters=%s: %s',
self.engine_cube.name,
self.name,
filters,
members,
)
return members
members = []
@ -184,7 +187,8 @@ class EngineDimension(object):
table_expression = '%s' % self.engine_cube.fact_table
if joins:
table_expression = self.engine_cube.build_table_expression(
joins, self.engine_cube.fact_table, force_join='right')
joins, self.engine_cube.fact_table, force_join='right'
)
sql = 'SELECT %s AS value, %s::text AS label ' % (value, value_label)
sql += 'FROM %s ' % table_expression
if order_by:
@ -199,7 +203,7 @@ class EngineDimension(object):
if order_value not in group_by:
group_by.append(order_value)
if conditions:
sql += 'WHERE %s ' % (' AND '.join(conditions))
sql += 'WHERE %s ' % ' AND '.join(conditions)
sql += 'GROUP BY %s ' % ', '.join(group_by)
sql += 'ORDER BY (%s) ' % ', '.join(order_by)
sql = sql.format(fact_table=self.engine_cube.fact_table)
@ -210,14 +214,15 @@ class EngineDimension(object):
continue
members.append(Member(id=row[0], label=force_text(row[1])))
cache.set(cache_key, members, 600)
self.engine.log.debug('MEMBERS: dimension %s.%s filters=%s: %s',
self.engine_cube.name, self.name, filters,
members)
self.engine.log.debug(
'MEMBERS: dimension %s.%s filters=%s: %s', self.engine_cube.name, self.name, filters, members
)
return members
class SchemaJSONDimension(schemas.Dimension):
'''Generated dimensions for JSON fields keys'''
filter = False
order_by = None
group_by = None
@ -225,7 +230,7 @@ class SchemaJSONDimension(schemas.Dimension):
type = 'string'
def __init__(self, json_field, name):
super(SchemaJSONDimension, self).__init__()
super().__init__()
name = str(name)
self.name = name
self.label = name.title()
@ -233,27 +238,30 @@ class SchemaJSONDimension(schemas.Dimension):
self.value_label = expr
self.value = expr
self.join = ['json_' + name]
sql = ('SELECT DISTINCT {json_field}->>\'%s\' AS v, {json_field}->>\'%s\' AS v'
' FROM {{fact_table}} WHERE ({json_field}->>\'%s\') IS NOT NULL ORDER BY v' %
(self.name, self.name, self.name))
sql = (
'SELECT DISTINCT {json_field}->>\'%s\' AS v, {json_field}->>\'%s\' AS v'
' FROM {{fact_table}} WHERE ({json_field}->>\'%s\') IS NOT NULL ORDER BY v'
% (self.name, self.name, self.name)
)
self.members_query = sql.format(json_field=json_field)
self.filter_expression = ('({fact_table}.id IS NULL '
'OR ({fact_table}.%s->>\'%s\') IN (%%s))'
% (json_field, name))
self.filter_expression = '({fact_table}.id IS NULL OR ({fact_table}.%s->>\'%s\') IN (%%s))' % (
json_field,
name,
)
self.filter_needs_join = False
self.absent_label = _('None')
class EngineJSONDimension(EngineDimension):
def __init__(self, engine, engine_cube, name):
self.engine = engine
self.engine_cube = engine_cube
self.dimension = SchemaJSONDimension(self.engine_cube.json_field, name)
def cache_key(self, filters):
key = (self.engine.path + self.engine_cube.json_field
+ self.engine_cube.name + self.name + repr(filters))
key = (
self.engine.path + self.engine_cube.json_field + self.engine_cube.name + self.name + repr(filters)
)
return hashlib.md5(force_bytes(key)).hexdigest()
def to_json(self):
@ -263,7 +271,7 @@ class EngineJSONDimension(EngineDimension):
}
class EngineMeasure(object):
class EngineMeasure:
def __init__(self, engine, engine_cube, measure):
self.engine = engine
self.engine_cube = engine_cube
@ -273,7 +281,7 @@ class EngineMeasure(object):
return getattr(self.measure, name)
class ProxyList(object):
class ProxyList:
chain = None
def __init__(self, engine, engine_cube, attribute, cls, chain=None):
@ -285,8 +293,9 @@ class ProxyList(object):
self.chain = chain(engine, engine_cube)
def __iter__(self):
i = (self.cls(self.engine, self.engine_cube, o)
for o in getattr(self.engine_cube.cube, self.attribute))
i = (
self.cls(self.engine, self.engine_cube, o) for o in getattr(self.engine_cube.cube, self.attribute)
)
if self.chain:
i = itertools.chain(i, self.chain)
return i
@ -300,7 +309,7 @@ class ProxyList(object):
raise KeyError
class JSONDimensions(object):
class JSONDimensions:
__cache = None
def __init__(self, engine, engine_cube):
@ -313,8 +322,10 @@ class JSONDimensions(object):
return []
if not self.__cache:
with self.engine.get_cursor() as cursor:
sql = ('select distinct jsonb_object_keys(%s) as a from %s order by a'
% (self.engine_cube.json_field, self.engine_cube.fact_table))
sql = 'select distinct jsonb_object_keys(%s) as a from %s order by a' % (
self.engine_cube.json_field,
self.engine_cube.fact_table,
)
cursor.execute(sql)
self.__cache = [row[0] for row in cursor.fetchall()]
return self.__cache
@ -329,7 +340,7 @@ class JSONDimensions(object):
return EngineJSONDimension(self.engine, self.engine_cube, name)
class ProxyListDescriptor(object):
class ProxyListDescriptor:
def __init__(self, attribute, cls, chain=None):
self.attribute = attribute
self.cls = cls
@ -342,7 +353,7 @@ class ProxyListDescriptor(object):
return obj.__dict__[key]
class EngineCube(object):
class EngineCube:
dimensions = ProxyListDescriptor('all_dimensions', EngineDimension, chain=JSONDimensions)
measures = ProxyListDescriptor('measures', EngineMeasure)
@ -365,10 +376,16 @@ class EngineCube(object):
name=name,
table=(
'(SELECT DISTINCT %s.%s->>\'%s\' AS value FROM %s '
'WHERE (%s.%s->>\'%s\') IS NOT NULL ORDER BY value)' % (
self.fact_table, self.json_field, json_key, self.fact_table,
self.fact_table, self.json_field, json_key
)
'WHERE (%s.%s->>\'%s\') IS NOT NULL ORDER BY value)'
)
% (
self.fact_table,
self.json_field,
json_key,
self.fact_table,
self.fact_table,
self.json_field,
json_key,
),
master='%s->>\'%s\'' % (self.json_field, json_key),
detail='value',
@ -427,15 +444,22 @@ class EngineCube(object):
sql += ' GROUP BY %s' % ', '.join(group_by)
if order_by:
sql += ' ORDER BY %s' % ', '.join(order_by)
sql = sql.format(fact_table=self.cube.fact_table,
table_expression=table_expression,
where_conditions=where_conditions)
sql = sql.format(
fact_table=self.cube.fact_table,
table_expression=table_expression,
where_conditions=where_conditions,
)
return sql
def query(self, filters, drilldown, measures, **kwargs):
self.engine.log.debug('%s.%s query filters=%s drilldown=%s measures=%s',
self.engine.warehouse.name, self.cube.name, filters, drilldown,
measures)
self.engine.log.debug(
'%s.%s query filters=%s drilldown=%s measures=%s',
self.engine.warehouse.name,
self.cube.name,
filters,
drilldown,
measures,
)
with self.engine.get_cursor() as cursor:
sql = self.sql_query(filters=filters, drilldown=drilldown, measures=measures, **kwargs)
self.engine.log.debug('SQL: %s', sql)
@ -451,16 +475,20 @@ class EngineCube(object):
else:
value_label = row[j + 1]
j += 2
cells.dimensions.append(DimensionCell(
dimension=dimension,
value=value,
value_label=value_label,
))
cells.dimensions.append(
DimensionCell(
dimension=dimension,
value=value,
value_label=value_label,
)
)
for i, measure in enumerate(measures):
cells.measures.append(MeasureCell(
measure=measure,
value=row[j + i],
))
cells.measures.append(
MeasureCell(
measure=measure,
value=row[j + i],
)
)
yield cells
JOIN_KINDS = {
@ -471,8 +499,8 @@ class EngineCube(object):
}
def build_table_expression(self, joins, table_name, force_join=None):
'''Recursively build the table expression from the join tree,
starting from the fact table'''
"""Recursively build the table expression from the join tree,
starting from the fact table"""
join_tree = {}
# Build join tree
@ -495,12 +523,15 @@ class EngineCube(object):
for join_name, join in joins.items():
contain_joins = True
sql += ' %s ' % self.JOIN_KINDS[kind]
sql += ' ' + build_table_expression_helper(join_tree, join.table, alias=join.name, top=False)
sql += ' ' + build_table_expression_helper(
join_tree, join.table, alias=join.name, top=False
)
condition = '%s.%s = %s.%s' % (
alias or table_name,
join.master.split('.')[-1],
quote(join.name),
join.detail)
join.detail,
)
sql += ' ON ' + condition
# if the table expression contains joins and is not the full table
@ -514,7 +545,7 @@ class EngineCube(object):
return build_table_expression_helper(join_tree, table_name)
class Engine(object):
class Engine:
def __init__(self, warehouse):
self.warehouse = warehouse
self.log = logging.getLogger(__name__)

View File

@ -25,12 +25,11 @@ try:
except ImportError:
sentry_sdk = None
from tenant_schemas.utils import tenant_context
from django.conf import settings
from django.utils.encoding import force_str
from hobo.agent.common.management.commands import hobo_deploy
from hobo.multitenant.settings_loaders import KnownServices
from django.utils.encoding import force_str
from django.conf import settings
from tenant_schemas.utils import tenant_context
def pg_dsn_quote(value):
@ -47,9 +46,10 @@ def truncate_pg_identifier(identifier, hash_length=6, force_hash=False):
else:
# insert hash in the middle, to keep some readability
return (
identifier[:(63 - hash_length) // 2]
identifier[: (63 - hash_length) // 2]
+ hashlib.md5(identifier.encode('utf-8')).hexdigest()[:hash_length]
+ identifier[-(63 - hash_length) // 2:])
+ identifier[-(63 - hash_length) // 2 :]
)
def schema_from_url(url, hash_length=6):
@ -62,7 +62,7 @@ def schema_from_url(url, hash_length=6):
class Command(hobo_deploy.Command):
def deploy_specifics(self, hobo_environment, tenant):
super(Command, self).deploy_specifics(hobo_environment, tenant)
super().deploy_specifics(hobo_environment, tenant)
with tenant_context(tenant):
services = hobo_environment.get('services')
ini_file = os.path.join(tenant.get_directory(), 'wcs-olap.ini')
@ -77,15 +77,18 @@ class Command(hobo_deploy.Command):
config.add_section('wcs-olap')
config.set('wcs-olap', 'cubes_model_dirs', schemas_path)
pg_dsn_parts = []
for pg_dsn_part in [('NAME', 'dbname'),
('HOST', 'host'),
('USER', 'user'),
('PASSWORD', 'password'),
('PORT', 'port')]:
for pg_dsn_part in [
('NAME', 'dbname'),
('HOST', 'host'),
('USER', 'user'),
('PASSWORD', 'password'),
('PORT', 'port'),
]:
if settings.DATABASES['default'].get(pg_dsn_part[0]):
pg_dsn_parts.append('%s=%s' % (
pg_dsn_part[1],
pg_dsn_quote(settings.DATABASES['default'].get(pg_dsn_part[0]))))
pg_dsn_parts.append(
'%s=%s'
% (pg_dsn_part[1], pg_dsn_quote(settings.DATABASES['default'].get(pg_dsn_part[0])))
)
config.set('wcs-olap', 'pg_dsn', config_parser_quote(' '.join(pg_dsn_parts)))
for service in services:
@ -97,8 +100,12 @@ class Command(hobo_deploy.Command):
our_key = this['secret_key']
for service in services:
base_url = service.get('base_url')
if (service.get('this') or not service.get('secret_key')
or service['service-id'] != 'wcs' or not service.get('base_url')):
if (
service.get('this')
or not service.get('secret_key')
or service['service-id'] != 'wcs'
or not service.get('base_url')
):
continue
elif service.get('secondary') and not config.has_section(base_url):
# skip secondary instances unless they were already added,

View File

@ -27,8 +27,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
'--output', metavar='FILE', default=None,
help='name of a file to write output to')
'--output', metavar='FILE', default=None, help='name of a file to write output to'
)
def handle(self, *args, **options):
if options['output']:

View File

@ -26,15 +26,11 @@ class Command(BaseCommand):
help = 'Import an exported site'
def add_arguments(self, parser):
parser.add_argument('filename', metavar='FILENAME', type=str, help='name of file to import')
parser.add_argument('--clean', action='store_true', default=False, help='Clean site before importing')
parser.add_argument(
'filename', metavar='FILENAME', type=str,
help='name of file to import')
parser.add_argument(
'--clean', action='store_true', default=False,
help='Clean site before importing')
parser.add_argument(
'--if-empty', action='store_true', default=False,
help='Import only if site is empty')
'--if-empty', action='store_true', default=False, help='Import only if site is empty'
)
def handle(self, filename, **options):
if filename == '-':

View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert
#
@ -17,23 +16,24 @@
import re
from datetime import date
from dateutil.relativedelta import relativedelta, MO
import isodate
from dateutil.relativedelta import MO, relativedelta
class RelativeDate(date):
__TEMPLATES = [
{
'pattern': u' *cette +année *',
'pattern': ' *cette +année *',
'truncate': 'year',
},
{
'pattern': u' *l\'année +dernière *',
'pattern': ' *l\'année +dernière *',
'truncate': 'year',
'timedelta': {'years': -1},
},
{
'pattern': u' *l\'année +prochaine *',
'pattern': ' *l\'année +prochaine *',
'truncate': 'year',
'timedelta': {'years': 1},
},
@ -131,7 +131,7 @@ class RelativeDate(date):
elif group.startswith('-'):
sign = -1
group = group[1:]
n = re.match('(\d+)\*(\w+)', group)
n = re.match(r'(\d+)\*(\w+)', group)
try:
if n:
value = int(n.group(1) * m.group(n.group(2)))

View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
#
# bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert
@ -16,9 +15,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import collections
import datetime
import decimal
import collections
from django.utils import six
from django.utils.encoding import force_text
@ -49,7 +48,7 @@ def type_cast(t):
return t
class Base(object):
class Base:
__types__ = {}
def __init__(self, **kwargs):
@ -78,14 +77,16 @@ class Base(object):
def from_json(cls, d):
assert hasattr(d, 'keys')
slots = cls.slots()
assert set(d.keys()) <= set(slots), \
'given keys %r does not match %s.__slots__: %r' % (d.keys(), cls.__name__, slots)
assert set(d.keys()) <= set(slots), 'given keys %r does not match %s.__slots__: %r' % (
d.keys(),
cls.__name__,
slots,
)
types = cls.types()
kwargs = {}
for key in slots:
assert key in d or hasattr(cls, key), \
'%s.%s is is a mandatory attribute' % (cls.__name__, key)
assert key in d or hasattr(cls, key), '%s.%s is is a mandatory attribute' % (cls.__name__, key)
if not key in d:
continue
value = d[key]
@ -133,7 +134,7 @@ class Measure(Base):
__slots__ = ['name', 'label', 'type', 'expression']
__types__ = {
'name': str,
'label': six.text_type,
'label': str,
'type': type_cast,
'expression': str,
}
@ -146,12 +147,25 @@ class Measure(Base):
class Dimension(Base):
__slots__ = ['name', 'label', 'type', 'join', 'value', 'value_label',
'order_by', 'group_by', 'filter_in_join', 'filter', 'filter_value',
'filter_needs_join', 'filter_expression', 'absent_label']
__slots__ = [
'name',
'label',
'type',
'join',
'value',
'value_label',
'order_by',
'group_by',
'filter_in_join',
'filter',
'filter_value',
'filter_needs_join',
'filter_expression',
'absent_label',
]
__types__ = {
'name': str,
'label': six.text_type,
'label': str,
'type': str,
'join': [str],
'value': str,
@ -164,7 +178,7 @@ class Dimension(Base):
'filter_in_join': bool,
'filter_value': str,
'filter_needs_join': bool,
'absent_label': six.text_type,
'absent_label': str,
}
def __init__(self, **kwargs):
@ -180,7 +194,7 @@ class Dimension(Base):
self.filter_needs_join = True
self.members_query = None
self.absent_label = None
super(Dimension, self).__init__(**kwargs)
super().__init__(**kwargs)
if not self.absent_label:
if self.type in ('date', 'integer', 'string'):
self.absent_label = _('None')
@ -196,33 +210,31 @@ class Dimension(Base):
return [
self,
Dimension(
label=u'année (%s)' % self.label,
label='année (%s)' % self.label,
name=self.name + '__year',
type='integer',
join=self.join,
filter_value='EXTRACT(year from %s)::integer' % filter_value,
filter_in_join=self.filter_in_join,
value='EXTRACT(year from %s)::integer' % self.value,
filter=False),
filter=False,
),
Dimension(
label=u'année et mois (%s)' % self.label,
label='année et mois (%s)' % self.label,
name=self.name + '__yearmonth',
type='integer',
join=self.join,
filter_value='EXTRACT(year from %s) || \'M\' || EXTRACT(month from %s)'
% (filter_value, filter_value),
% (filter_value, filter_value),
filter_in_join=self.filter_in_join,
value='TO_CHAR(EXTRACT(month from %s), \'00\') || \'/\' || EXTRACT(year from %s)'
% (self.value, self.value),
group_by='EXTRACT(year from %s), EXTRACT(month from %s)' % (self.value,
self.value),
order_by=['EXTRACT(year from %s), EXTRACT(month from %s)' % (self.value,
self.value)],
filter=False),
% (self.value, self.value),
group_by='EXTRACT(year from %s), EXTRACT(month from %s)' % (self.value, self.value),
order_by=['EXTRACT(year from %s), EXTRACT(month from %s)' % (self.value, self.value)],
filter=False,
),
Dimension(
label=u'mois (%s)' % self.label,
label='mois (%s)' % self.label,
name=self.name + '__month',
type='integer',
filter_value='EXTRACT(month from %s)' % filter_value,
@ -230,12 +242,12 @@ class Dimension(Base):
join=self.join,
value='EXTRACT(month from %s)' % self.value,
value_label='to_char(date_trunc(\'month\', %s), \'TMmonth\')' % self.value,
group_by='EXTRACT(month from %s), '
'to_char(date_trunc(\'month\', %s), \'TMmonth\')'
% (self.value, self.value),
filter=False),
group_by='EXTRACT(month from %s), to_char(date_trunc(\'month\', %s), \'TMmonth\')'
% (self.value, self.value),
filter=False,
),
Dimension(
label=u'jour de la semaine (%s)' % self.label,
label='jour de la semaine (%s)' % self.label,
name=self.name + '__dow',
type='integer',
join=self.join,
@ -243,24 +255,27 @@ class Dimension(Base):
filter_in_join=self.filter_in_join,
value='EXTRACT(dow from %s)' % self.value,
order_by=['(EXTRACT(dow from %s) + 6)::integer %% 7' % self.value],
value_label='to_char(date_trunc(\'week\', current_date)::date '
'+ EXTRACT(dow from %s)::integer - 1, \'TMday\')' % self.value,
filter=False),
value_label=(
'to_char(date_trunc(\'week\', current_date)::date '
'+ EXTRACT(dow from %s)::integer - 1, \'TMday\')'
)
% self.value,
filter=False,
),
Dimension(
label=u'semaine (%s)' % self.label,
label='semaine (%s)' % self.label,
name=self.name + '__isoweek',
type='integer',
join=self.join,
filter_value='EXTRACT(isoyear from %s) || \'S\' || EXTRACT(week from %s)'
% (filter_value, filter_value),
% (filter_value, filter_value),
filter_in_join=self.filter_in_join,
value='EXTRACT(isoyear from %s) || \'S\' || EXTRACT(week from %s)'
% (self.value, self.value),
group_by='EXTRACT(isoyear from %s), EXTRACT(week from %s)' % (self.value,
self.value),
order_by=['EXTRACT(isoyear from %s), EXTRACT(week from %s)' % (self.value,
self.value)],
filter=False)
% (self.value, self.value),
group_by='EXTRACT(isoyear from %s), EXTRACT(week from %s)' % (self.value, self.value),
order_by=['EXTRACT(isoyear from %s), EXTRACT(week from %s)' % (self.value, self.value)],
filter=False,
),
]
return [self]
@ -268,8 +283,7 @@ class Dimension(Base):
value = self.filter_value or self.value
if self.type == 'date':
assert isinstance(filter_values, dict) and set(filter_values.keys()) == set(['start',
'end'])
assert isinstance(filter_values, dict) and set(filter_values.keys()) == {'start', 'end'}
filters = []
values = []
@ -278,6 +292,7 @@ class Dimension(Base):
filter_value = RelativeDate(filter_value)
filters.append(tpl % (value, '%s'))
values.append(filter_value)
try:
if filter_values['start']:
date_filter('%s >= %s', filter_values['start'])
@ -346,7 +361,7 @@ class Join(Base):
def __init__(self, **kwargs):
self.kind = 'full'
super(Join, self).__init__(**kwargs)
super().__init__(**kwargs)
@property
def master_table(self):
@ -356,18 +371,27 @@ class Join(Base):
class Cube(Base):
__slots__ = ['name', 'label', 'fact_table', 'json_field', 'key', 'joins', 'dimensions',
'measures', 'warnings']
__slots__ = [
'name',
'label',
'fact_table',
'json_field',
'key',
'joins',
'dimensions',
'measures',
'warnings',
]
__types__ = {
'name': str,
'label': six.text_type,
'label': str,
'fact_table': str,
'json_field': str,
'key': str,
'joins': [Join],
'dimensions': [Dimension],
'measures': [Measure],
'warnings': [six.text_type],
'warnings': [str],
}
def __init__(self, **kwargs):
@ -376,7 +400,7 @@ class Cube(Base):
self.dimensions = ()
self.measures = ()
self.warnings = ()
super(Cube, self).__init__(**kwargs)
super().__init__(**kwargs)
def check(self):
names = collections.Counter()
@ -386,13 +410,13 @@ class Cube(Base):
duplicates = [k for k, v in names.items() if v > 1]
if duplicates:
raise SchemaError(
'More than one join, dimension or measure with name(s) %s' % ', '.join(duplicates))
'More than one join, dimension or measure with name(s) %s' % ', '.join(duplicates)
)
@property
def all_dimensions(self):
for dimension in self.dimensions:
for sub_dimension in dimension.dimensions:
yield sub_dimension
yield from dimension.dimensions
def get_dimension(self, name):
for dimension in self.dimensions:
@ -419,7 +443,7 @@ class Warehouse(Base):
__types__ = {
'name': str,
'slug': str,
'label': six.text_type,
'label': str,
'pg_dsn': str,
'search_path': [str],
'cubes': [Cube],
@ -430,7 +454,7 @@ class Warehouse(Base):
self.path = None
self.slug = None
self.timestamp = None
super(Warehouse, self).__init__(**kwargs)
super().__init__(**kwargs)
def check(self):
names = collections.Counter(cube.name for cube in self.cubes)

View File

@ -24,14 +24,14 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""
from itertools import chain
from django.conf import global_settings
import django
from gadjo.templatetags.gadjo import xstatic
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from itertools import chain
import django
from django.conf import global_settings
from gadjo.templatetags.gadjo import xstatic
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
@ -74,8 +74,7 @@ MIDDLEWARE = (
'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
STATICFILES_FINDERS = list(chain(global_settings.STATICFILES_FINDERS,
('gadjo.finders.XStaticFinder',)))
STATICFILES_FINDERS = list(chain(global_settings.STATICFILES_FINDERS, ('gadjo.finders.XStaticFinder',)))
ROOT_URLCONF = 'bijoe.urls'
@ -122,7 +121,7 @@ LOGGING = {
'formatters': {
'verbose': {
'format': '[%(asctime)s] %(levelname)s %(name)s.%(funcName)s: %(message)s',
'datefmt': '%Y-%m-%d %a %H:%M:%S'
'datefmt': '%Y-%m-%d %a %H:%M:%S',
},
},
'handlers': {

View File

@ -21,7 +21,7 @@ register = template.Library()
try:
from django_select2.templatetags.django_select2_tags import *
except ImportError:
@register.simple_tag(name='import_django_select2_js_css')
def import_all(light=0):
return ''

View File

@ -14,9 +14,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.conf import settings
from django.conf.urls import include, url
from django.contrib import admin
from django.conf import settings
from . import views

View File

@ -15,14 +15,14 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
import os
import glob
import json
import os
from django.conf import settings
from django.db import connection, transaction
from django.utils.translation import ugettext as _
from django.utils.timezone import utc
from django.utils.translation import ugettext as _
try:
from functools import lru_cache
@ -34,12 +34,10 @@ from .schemas import Warehouse
def get_warehouses_paths():
for pattern in settings.BIJOE_SCHEMAS:
for path in glob.glob(pattern):
yield path
yield from glob.glob(pattern)
if hasattr(connection, 'tenant'):
pattern = os.path.join(connection.tenant.get_directory(), 'schemas', '*.model')
for path in glob.glob(pattern):
yield path
yield from glob.glob(pattern)
@lru_cache()
@ -70,8 +68,8 @@ def human_join(l):
if len(l) == 1:
return l[0]
if len(l) > 2:
l = u', '.join(l[:-1]), l[-1]
return _(u'{0} and {1}').format(l[0], l[1])
l = ', '.join(l[:-1]), l[-1]
return _('{0} and {1}').format(l[0], l[1])
def export_site():

View File

@ -21,7 +21,6 @@ import os
import subprocess
from django.conf import settings
from uwsgidecorators import cron, spool
# existing loggers are disabled by Django on django.setup() due do
@ -58,7 +57,8 @@ def launch_wcs_olap(wcs_olap_ini_path):
'--all',
wcs_olap_ini_path,
],
check=False)
check=False,
)
logger.info('finished wcs-olap on %s', wcs_olap_ini_path)

View File

@ -17,31 +17,31 @@
import json
from django.conf import settings
from django.shortcuts import resolve_url
from django.urls import reverse
from django.views.generic import ListView, View
from django.http import HttpResponse, HttpResponseRedirect
from django.utils.decorators import method_decorator
from django.utils.http import quote
from django.utils.translation import ugettext as _
from django.contrib.auth import logout as auth_logout
from django.contrib.auth import views as auth_views
from django.contrib.auth.views import redirect_to_login
from django.core.exceptions import PermissionDenied
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import resolve_url
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.utils.http import quote
from django.utils.translation import ugettext as _
from django.views.decorators.cache import never_cache
from django.views.generic import ListView, View
try:
from mellon.utils import get_idps
except ImportError:
get_idps = lambda: []
from .utils import get_warehouses
from .engine import Engine
from .utils import get_warehouses
from .visualization.models import Visualization
from .visualization.utils import Visualization as VisuUtil
class AuthorizationMixin(object):
class AuthorizationMixin:
def authorize(self, request):
if request.user.is_authenticated:
if not request.user.is_superuser:
@ -52,7 +52,7 @@ class AuthorizationMixin(object):
def dispatch(self, request, *args, **kwargs):
if self.authorize(request):
return super(AuthorizationMixin, self).dispatch(request, *args, **kwargs)
return super().dispatch(request, *args, **kwargs)
else:
return redirect_to_login(request.build_absolute_uri())
@ -64,9 +64,8 @@ class HomepageView(AuthorizationMixin, ListView):
paginate_by = settings.PAGE_LENGTH
def get_context_data(self, **kwargs):
ctx = super(HomepageView, self).get_context_data(**kwargs)
ctx['warehouses'] = sorted((Engine(w) for w in get_warehouses()),
key=lambda w: w.label)
ctx = super().get_context_data(**kwargs)
ctx['warehouses'] = sorted((Engine(w) for w in get_warehouses()), key=lambda w: w.label)
ctx['request'] = self.request
return ctx
@ -108,7 +107,7 @@ class LoginView(auth_views.LoginView):
return HttpResponseRedirect(
resolve_url('mellon_login') + '?next=' + quote(request.GET.get('next'))
)
return super(LoginView, self).get(request, *args, **kwargs)
return super().get(request, *args, **kwargs)
login = LoginView.as_view()
@ -119,7 +118,7 @@ class LogoutView(auth_views.LogoutView):
def dispatch(self, request, *args, **kwargs):
if any(get_idps()):
return HttpResponseRedirect(resolve_url('mellon_logout'))
return super(LogoutView, self).dispatch(request, *args, **kwargs)
return super().dispatch(request, *args, **kwargs)
logout = LogoutView.as_view()

View File

@ -22,4 +22,5 @@ from . import models
class VisualizationAdmin(admin.ModelAdmin):
list_display = ['name']
admin.site.register(models.Visualization, VisualizationAdmin)

View File

@ -1,4 +1,3 @@
# -*- encoding: utf-8 -*-
# bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert
#
@ -17,13 +16,12 @@
from django import forms
from django.core.exceptions import ValidationError
from django.utils.encoding import force_text
from django.utils.translation import ugettext as _
from django.utils.safestring import mark_safe
from django.forms import ModelForm, TextInput, NullBooleanField
from django.conf import settings
from django.core.exceptions import ValidationError
from django.forms import ModelForm, NullBooleanField, TextInput
from django.utils.encoding import force_text
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _
from django_select2.forms import HeavySelect2MultipleWidget
from . import models
@ -32,54 +30,59 @@ from . import models
class VisualizationForm(ModelForm):
class Meta:
model = models.Visualization
exclude = ('slug', 'parameters',)
exclude = (
'slug',
'parameters',
)
widgets = {
'name': TextInput,
}
DATE_RANGES = [
{
'value': '3_last_months',
'label': _('3 last months'),
'start': u"les 3 derniers mois",
'end': u"maintenant",
'start': "les 3 derniers mois",
'end': "maintenant",
},
{
'value': 'this_year',
'label': _('this year'),
'start': u"cette année",
'end': u"l\'année prochaine",
'start': "cette année",
'end': "l\'année prochaine",
},
{
'value': 'last_year',
'label': _('last year'),
'start': u'l\'année dernière',
'end': u'cette année',
'start': 'l\'année dernière',
'end': 'cette année',
},
{
'value': 'this_quarter',
'label': _('this quarter'),
'start': u'ce trimestre',
'start': 'ce trimestre',
'end': "le prochain trimestre",
},
{
'value': 'last_quarter',
'label': _('last quarter'),
'start': u'le dernier trimestre',
'end': u'ce trimestre',
'start': 'le dernier trimestre',
'end': 'ce trimestre',
},
{
'value': 'since_1jan_last_year',
'label': _('since 1st january last year'),
'start': u'l\'année dernière',
'end': u'maintenant',
'start': 'l\'année dernière',
'end': 'maintenant',
},
]
def get_date_range_choices():
return [('', '---')] + [(r['value'], r['label'])
for r in getattr(settings, 'BIJOE_DATE_RANGES', DATE_RANGES)]
return [('', '---')] + [
(r['value'], r['label']) for r in getattr(settings, 'BIJOE_DATE_RANGES', DATE_RANGES)
]
class DateRangeWidget(forms.MultiWidget):
@ -87,15 +90,15 @@ class DateRangeWidget(forms.MultiWidget):
attrs = attrs.copy() if attrs else {}
attrs.update({'type': 'date', 'autocomplete': 'off'})
attrs1 = attrs.copy()
attrs1['placeholder'] = _(u'start')
attrs1['placeholder'] = _('start')
attrs2 = attrs.copy()
attrs2['placeholder'] = _(u'end')
attrs2['placeholder'] = _('end')
widgets = (
forms.DateInput(attrs=attrs1, format='%Y-%m-%d'),
forms.DateInput(attrs=attrs2, format='%Y-%m-%d'),
forms.Select(choices=get_date_range_choices()),
)
super(DateRangeWidget, self).__init__(widgets, attrs=attrs)
super().__init__(widgets, attrs=attrs)
def decompress(self, value):
if not value:
@ -106,11 +109,10 @@ class DateRangeWidget(forms.MultiWidget):
return value['start'], value['end'], None
def render(self, name, value, attrs=None, renderer=None):
output = super(DateRangeWidget, self).render(name, value, attrs=attrs)
output = super().render(name, value, attrs=attrs)
_id = self.build_attrs(attrs).get('id', None)
if _id:
output += mark_safe("<script>$(function () { bijoe_date_range('#%s'); });</script>" %
_id)
output += mark_safe("<script>$(function () { bijoe_date_range('#%s'); });</script>" % _id)
return output
class Media:
@ -125,10 +127,9 @@ class DateRangeField(forms.MultiValueField):
fields = (
forms.DateField(required=False),
forms.DateField(required=False),
forms.ChoiceField(choices=get_date_range_choices(), required=False)
forms.ChoiceField(choices=get_date_range_choices(), required=False),
)
super(DateRangeField, self).__init__(fields=fields, require_all_fields=False, *args,
**kwargs)
super().__init__(fields=fields, require_all_fields=False, *args, **kwargs)
def compress(self, values):
if not values:
@ -159,10 +160,10 @@ class Select2ChoicesWidget(HeavySelect2MultipleWidget):
class CubeForm(forms.Form):
representation = forms.ChoiceField(
label=_(u'Presentation'),
choices=[('table', _('table')),
('graphical', _('chart'))],
widget=forms.RadioSelect())
label=_('Presentation'),
choices=[('table', _('table')), ('graphical', _('chart'))],
widget=forms.RadioSelect(),
)
def __init__(self, *args, **kwargs):
self.cube = cube = kwargs.pop('cube')
@ -170,12 +171,13 @@ class CubeForm(forms.Form):
dimension_choices = [('', '')] + [
(dimension.name, dimension.label)
for dimension in cube.dimensions if dimension.type not in ('datetime', 'date')]
for dimension in cube.dimensions
if dimension.type not in ('datetime', 'date')
]
# loop
self.base_fields['loop'] = forms.ChoiceField(
label=_('Loop by'),
choices=dimension_choices,
required=False)
label=_('Loop by'), choices=dimension_choices, required=False
)
# filters
for dimension in cube.dimensions:
@ -184,10 +186,12 @@ class CubeForm(forms.Form):
field_name = 'filter__%s' % dimension.name
if dimension.type == 'date':
self.base_fields[field_name] = DateRangeField(
label=dimension.label.capitalize(), required=False)
label=dimension.label.capitalize(), required=False
)
elif dimension.type == 'bool':
self.base_fields[field_name] = NullBooleanField(
label=dimension.label.capitalize(), required=False)
label=dimension.label.capitalize(), required=False
)
else:
members = []
for _id, label in dimension.members():
@ -200,6 +204,7 @@ class CubeForm(forms.Form):
if v == s:
return value
return None
return f
self.base_fields[field_name] = forms.TypedMultipleChoiceField(
@ -211,38 +216,34 @@ class CubeForm(forms.Form):
data_view='select2-choices',
warehouse=cube.engine.warehouse.name,
cube=cube.name,
dimension=dimension.name
))
dimension=dimension.name,
),
)
# group by
self.base_fields['drilldown_x'] = forms.ChoiceField(
label=_('Group by - horizontally'),
choices=dimension_choices,
required=False)
label=_('Group by - horizontally'), choices=dimension_choices, required=False
)
self.base_fields['drilldown_y'] = forms.ChoiceField(
label=_('Group by - vertically'),
choices=dimension_choices,
required=False)
label=_('Group by - vertically'), choices=dimension_choices, required=False
)
# measures
choices = [(measure.name, measure.label)
for measure in cube.measures if measure.type != 'point']
self.base_fields['measure'] = forms.ChoiceField(
label=_('Measure'), choices=choices)
choices = [(measure.name, measure.label) for measure in cube.measures if measure.type != 'point']
self.base_fields['measure'] = forms.ChoiceField(label=_('Measure'), choices=choices)
super(CubeForm, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
def clean(self):
cleaned_data = super(CubeForm, self).clean()
cleaned_data = super().clean()
loop = cleaned_data.get('loop')
drilldown_x = cleaned_data.get('drilldown_x')
drilldown_y = cleaned_data.get('drilldown_y')
if loop and (loop == drilldown_x or loop == drilldown_y):
raise ValidationError({'loop': _('You cannot use the same dimension for looping and'
' grouping')})
raise ValidationError({'loop': _('You cannot use the same dimension for looping and grouping')})
return cleaned_data

View File

@ -1,20 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import jsonfield.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
]
dependencies = []
operations = [
migrations.CreateModel(
name='Visualization',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
(
'id',
models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True),
),
('name', models.TextField(verbose_name='name')),
('parameters', jsonfield.fields.JSONField(default=dict, verbose_name='parameters')),
],

View File

@ -1,6 +1,3 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations

View File

@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.12 on 2019-03-28 07:17
from __future__ import unicode_literals
from django.db import migrations, models
from django.utils.text import slugify
@ -38,5 +36,5 @@ class Migration(migrations.Migration):
name='slug',
field=models.SlugField(null=True, unique=True, verbose_name='Identifier'),
),
migrations.RunPython(forward_func, reverse_func)
migrations.RunPython(forward_func, reverse_func),
]

View File

@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.12 on 2019-03-28 07:25
from __future__ import unicode_literals
from django.db import migrations, models

View File

@ -14,8 +14,8 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import datetime
import json
from django.db import models
from django.http import Http404
@ -57,19 +57,12 @@ class Visualization(models.Model):
return (self.slug,)
def export_json(self):
visualization = {
'slug': self.slug,
'name': self.name,
'parameters': self.parameters
}
visualization = {'slug': self.slug, 'name': self.name, 'parameters': self.parameters}
return visualization
@classmethod
def import_json(cls, data):
defaults = {
'name': data['name'],
'parameters': data['parameters']
}
defaults = {'name': data['name'], 'parameters': data['parameters']}
_, created = cls.objects.update_or_create(slug=data['slug'], defaults=defaults)
return created
@ -85,7 +78,7 @@ class Visualization(models.Model):
i += 1
slug = '%s-%s' % (base_slug, i)
self.slug = slug
return super(Visualization, self).save(*args, **kwargs)
return super().save(*args, **kwargs)
@property
def exists(self):

View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
#
# bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert
@ -17,13 +16,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
import zipfile
import xml.etree.ElementTree as ET
import zipfile
from django.utils.encoding import force_text
OFFICE_NS = 'urn:oasis:names:tc:opendocument:xmlns:office:1.0'
TABLE_NS = 'urn:oasis:names:tc:opendocument:xmlns:table:1.0'
TEXT_NS = 'urn:oasis:names:tc:opendocument:xmlns:text:1.0'
@ -31,13 +28,10 @@ XLINK_NS = 'http://www.w3.org/1999/xlink'
def is_number(x):
if sys.version_info >= (3, 0):
return isinstance(x, (int, float))
else:
return isinstance(x, (int, long, float))
return isinstance(x, (int, float))
class Workbook(object):
class Workbook:
def __init__(self, encoding='utf-8'):
self.sheets = []
self.encoding = encoding
@ -64,21 +58,27 @@ class Workbook(object):
z = zipfile.ZipFile(output, 'w')
z.writestr('content.xml', self.get_data())
z.writestr('mimetype', 'application/vnd.oasis.opendocument.spreadsheet')
z.writestr('META-INF/manifest.xml', '''<?xml version="1.0" encoding="UTF-8"?>
z.writestr(
'META-INF/manifest.xml',
'''<?xml version="1.0" encoding="UTF-8"?>
<manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0">
<manifest:file-entry manifest:full-path="/" manifest:media-type="application/vnd.oasis.opendocument.spreadsheet"/>
<manifest:file-entry manifest:full-path="styles.xml" manifest:media-type="text/xml"/>
<manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/>
<manifest:file-entry manifest:full-path="META-INF/manifest.xml" manifest:media-type="text/xml"/>
<manifest:file-entry manifest:full-path="mimetype" manifest:media-type="text/plain"/>
</manifest:manifest>''')
z.writestr('styles.xml', '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
</manifest:manifest>''',
)
z.writestr(
'styles.xml',
'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<office:document-styles xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0">
</office:document-styles>''')
</office:document-styles>''',
)
z.close()
class WorkSheet(object):
class WorkSheet:
def __init__(self, workbook, name):
self.cells = {}
self.name = name
@ -104,7 +104,7 @@ class WorkSheet(object):
return root
class WorkCell(object):
class WorkCell:
def __init__(self, worksheet, value, hint=None):
self.value_type = 'string'
if is_number(value):

View File

@ -14,13 +14,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
import base64
import hmac
import datetime
import hashlib
import urllib
import random
import hmac
import logging
import random
import urllib
from django.utils import six
from django.utils.encoding import force_bytes, smart_bytes
@ -46,10 +46,7 @@ def sign_query(query, key, algo='sha256', timestamp=None, nonce=None):
new_query = query
if new_query:
new_query += '&'
new_query += urlencode((
('algo', algo),
('timestamp', timestamp),
('nonce', nonce)))
new_query += urlencode((('algo', algo), ('timestamp', timestamp), ('nonce', nonce)))
signature = base64.b64encode(sign_string(new_query, key, algo=algo))
new_query += '&signature=' + quote(signature)
return new_query
@ -71,7 +68,8 @@ def check_query(query, key, known_nonce=None, timedelta=30):
if not res:
key_hash = 'md5:%s' % hashlib.md5(force_bytes(key)).hexdigest()[:6]
logging.getLogger(__name__).warning(
'could not check signature of query %r with key %s: %s', query, key_hash, error)
'could not check signature of query %r with key %s: %s', query, key_hash, error
)
return res
@ -112,10 +110,6 @@ def check_string(s, signature, key, algo='sha256'):
if len(signature2) != len(signature):
return False
res = 0
if six.PY3:
for a, b in zip(signature, signature2):
res |= a ^ b
else:
for a, b in zip(signature, signature2):
res |= ord(a) ^ ord(b)
for a, b in zip(signature, signature2):
res |= a ^ b
return res == 0

View File

@ -19,20 +19,18 @@ from django.conf.urls import url
from . import views
urlpatterns = [
url(r'^$',
views.visualizations, name='visualizations'),
url(r'^json/$',
views.visualizations_json, name='visualizations-json'),
url(r'^import/$',
views.visualizations_import, name='visualizations-import'),
url(r'^export$',
views.visualizations_export, name='visualizations-export'),
url(r'^$', views.visualizations, name='visualizations'),
url(r'^json/$', views.visualizations_json, name='visualizations-json'),
url(r'^import/$', views.visualizations_import, name='visualizations-import'),
url(r'^export$', views.visualizations_export, name='visualizations-export'),
url(r'^warehouse/(?P<warehouse>[^/]*)/$', views.warehouse, name='warehouse'),
url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/$', views.cube, name='cube'),
url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/iframe/$', views.cube_iframe,
name='cube-iframe'),
url(r'warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/save/$',
views.create_visualization, name='create-visualization'),
url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/iframe/$', views.cube_iframe, name='cube-iframe'),
url(
r'warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/save/$',
views.create_visualization,
name='create-visualization',
),
url(r'(?P<pk>\d+)/$', views.visualization, name='visualization'),
url(r'(?P<pk>\d+)/json/$', views.visualization_json, name='visualization-json'),
url(r'(?P<pk>\d+)/geojson/$', views.visualization_geojson, name='visualization-geojson'),

View File

@ -14,31 +14,31 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import re
import json
import hashlib
import collections
import copy
import datetime
import decimal
import copy
import collections
import hashlib
import json
import re
from django.conf import settings
from django.core.cache import cache
from django.http import Http404
from django.utils.encoding import force_bytes, force_text
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from django.http import Http404
from django.conf import settings
from ..engine import Engine, MeasureCell, Member
from ..utils import get_warehouses
from ..engine import Engine, Member, MeasureCell
from .ods import Workbook
class Visualization(object):
def __init__(self, cube, representation, measure, drilldown_x=None, drilldown_y=None,
filters=None, loop=None):
class Visualization:
def __init__(
self, cube, representation, measure, drilldown_x=None, drilldown_y=None, filters=None, loop=None
):
self.cube = cube
self.representation = representation
@ -74,9 +74,15 @@ class Visualization(object):
}
def copy(self):
return Visualization(self.cube, self.representation, measure=self.measure,
drilldown_x=self.drilldown_x, drilldown_y=self.drilldown_y,
filters=copy.deepcopy(self.filters), loop=self.loop)
return Visualization(
self.cube,
self.representation,
measure=self.measure,
drilldown_x=self.drilldown_x,
drilldown_y=self.drilldown_y,
filters=copy.deepcopy(self.filters),
loop=self.loop,
)
@staticmethod
def get_cube(d, warehouses=None):
@ -107,8 +113,15 @@ class Visualization(object):
loop = d.get('loop')
if loop:
loop = cube.dimensions[loop]
return cls(cube, representation, measure, drilldown_x=drilldown_x, drilldown_y=drilldown_y,
filters=filters, loop=loop)
return cls(
cube,
representation,
measure,
drilldown_x=drilldown_x,
drilldown_y=drilldown_y,
filters=filters,
loop=loop,
)
@classmethod
def from_form(cls, cube, form):
@ -126,11 +139,15 @@ class Visualization(object):
drilldown_y = drilldown_y and cube.dimensions[drilldown_y]
loop = cleaned_data.get('loop')
loop = loop and cube.dimensions[loop]
return cls(cube, cleaned_data['representation'],
measure,
drilldown_x=drilldown_x,
drilldown_y=drilldown_y,
filters=filters, loop=loop)
return cls(
cube,
cleaned_data['representation'],
measure,
drilldown_x=drilldown_x,
drilldown_y=drilldown_y,
filters=filters,
loop=loop,
)
@property
def key(self):
@ -147,21 +164,20 @@ class Visualization(object):
keys.append('$'.join([kw] + sorted(map(force_text, value))))
else:
# scalar values
keys.append(u'%s$%s' % (kw, force_text(value)))
keys.append('%s$%s' % (kw, force_text(value)))
keys += [dim.name for dim in self.drilldown]
keys += [self.measure.name]
key = '$'.join(v.encode('utf8') for v in keys)
return hashlib.md5(force_bytes(key)).hexdigest()
def data(self):
'''Execute aggregation query, list members and check None values in
dimensions.
'''
rows = list(self.cube.query(self.filters.items(),
self.drilldown,
[self.measure]))
self.members = {dimension: list(dimension.members(filters=self.filters.items()))
for dimension in self.drilldown}
"""Execute aggregation query, list members and check None values in
dimensions.
"""
rows = list(self.cube.query(self.filters.items(), self.drilldown, [self.measure]))
self.members = {
dimension: list(dimension.members(filters=self.filters.items())) for dimension in self.drilldown
}
seen_none = set()
for cells in rows:
# Keep "empty" dimension value if there is a non-zero measure associated
@ -268,10 +284,12 @@ class Visualization(object):
y_axis, grid = self.table_1d()
table.append([self.drilldown_y.label, self.measure.label])
for y in y_axis:
table.append([
y.label,
'%s' % (grid[y.id],),
])
table.append(
[
y.label,
'%s' % (grid[y.id],),
]
)
else:
table.append([self.measure.label, '%s' % (self.data()[0].measures[0],)])
@ -296,12 +314,16 @@ class Visualization(object):
if isinstance(value, decimal.Decimal):
value = float(value)
if isinstance(value, datetime.timedelta):
value = value.days + value.seconds / 86400.
value = value.days + value.seconds / 86400.0
return value
if len(self.drilldown) == 2:
(x_axis, y_axis), grid = self.table_2d()
cells = ((['%s' % x.label, '%s' % y.label], cell_value(grid[(x.id, y.id)])) for x in x_axis for y in y_axis)
cells = (
(['%s' % x.label, '%s' % y.label], cell_value(grid[(x.id, y.id)]))
for x in x_axis
for y in y_axis
)
elif len(self.drilldown) == 1:
axis, grid = self.table_1d()
cells = ((['%s' % x.label], cell_value(grid[x.id])) for x in axis)
@ -315,10 +337,12 @@ class Visualization(object):
raise NotImplementedError
for coords, value in cells:
json_data.append({
'coords': [{'value': coord} for coord in coords],
'measures': [{'value': value}],
})
json_data.append(
{
'coords': [{'value': coord} for coord in coords],
'measures': [{'value': value}],
}
)
return json_data
@ -350,7 +374,7 @@ class Visualization(object):
l.append(self.drilldown_y.label)
if self.loop:
l.append(self.loop.label)
return u', '.join(l)
return ', '.join(l)
def __iter__(self):
if self.loop:

View File

@ -14,47 +14,48 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import hashlib
import json
from django.conf import settings
from django.core import signing
from django.core.signing import BadSignature
from django.contrib import messages
from django.core import signing
from django.core.exceptions import PermissionDenied
from django.core.signing import BadSignature
from django.http import Http404, HttpResponse, JsonResponse
from django.shortcuts import redirect
from django.urls import reverse, reverse_lazy
from django.utils.encoding import force_bytes, force_text
from django.utils.text import slugify
from django.utils.timezone import now
from django.utils.translation import ungettext, ugettext as _
from django.views.generic.edit import CreateView, DeleteView, UpdateView, FormView
from django.views.generic.list import MultipleObjectMixin
from django.views.generic import DetailView, ListView, View, TemplateView
from django.shortcuts import redirect
from django.urls import reverse, reverse_lazy
from django.http import HttpResponse, Http404, JsonResponse
from django.core.exceptions import PermissionDenied
from django.utils.translation import ugettext as _
from django.utils.translation import ungettext
from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.generic import DetailView, ListView, TemplateView, View
from django.views.generic.edit import CreateView, DeleteView, FormView, UpdateView
from django.views.generic.list import MultipleObjectMixin
from django_select2.cache import cache
from rest_framework import generics
from rest_framework.response import Response
from bijoe.utils import get_warehouses, import_site, export_site
from ..engine import Engine
from . import models, forms, signature
from .utils import Visualization
from bijoe.utils import export_site, get_warehouses, import_site
from .. import views
from ..engine import Engine
from . import forms, models, signature
from .utils import Visualization
class WarehouseView(views.AuthorizationMixin, TemplateView):
template_name = 'bijoe/warehouse.html'
def get_context_data(self, **kwargs):
ctx = super(WarehouseView, self).get_context_data(**kwargs)
ctx = super().get_context_data(**kwargs)
try:
warehouse = [warehouse for warehouse in get_warehouses()
if warehouse.name == self.kwargs['warehouse']][0]
warehouse = [
warehouse for warehouse in get_warehouses() if warehouse.name == self.kwargs['warehouse']
][0]
except IndexError:
raise Http404
ctx['warehouse'] = Engine(warehouse)
@ -63,16 +64,16 @@ class WarehouseView(views.AuthorizationMixin, TemplateView):
return ctx
class CubeDisplayMixin(object):
class CubeDisplayMixin:
def get_context_data(self, **kwargs):
ctx = super(CubeDisplayMixin, self).get_context_data(**kwargs)
ctx = super().get_context_data(**kwargs)
ctx['warehouse'] = self.warehouse
ctx['cube'] = self.cube
ctx['visualization'] = self.visualization
return ctx
class CubeMixin(object):
class CubeMixin:
def visualization(self, request, cube):
self.form = forms.CubeForm(cube=self.cube, data=request.GET or request.POST)
if self.form.is_valid():
@ -80,8 +81,9 @@ class CubeMixin(object):
def dispatch(self, request, *args, **kwargs):
try:
self.warehouse = Engine([warehouse for warehouse in get_warehouses()
if warehouse.name == self.kwargs['warehouse']][0])
self.warehouse = Engine(
[warehouse for warehouse in get_warehouses() if warehouse.name == self.kwargs['warehouse']][0]
)
except IndexError:
raise Http404
try:
@ -89,7 +91,7 @@ class CubeMixin(object):
except KeyError:
raise Http404
self.visualization = self.visualization(request, cube)
return super(CubeMixin, self).dispatch(request, *args, **kwargs)
return super().dispatch(request, *args, **kwargs)
class CubeView(views.AuthorizationMixin, CubeDisplayMixin, CubeMixin, TemplateView):
@ -101,7 +103,7 @@ class CubeView(views.AuthorizationMixin, CubeDisplayMixin, CubeMixin, TemplateVi
return self.get(request, *args, **kwargs)
def get_context_data(self, **kwargs):
ctx = super(CubeView, self).get_context_data(**kwargs)
ctx = super().get_context_data(**kwargs)
ctx['form'] = self.form
return ctx
@ -115,13 +117,13 @@ class CreateVisualizationView(views.AuthorizationMixin, CubeMixin, CreateView):
def get(self, request, *args, **kwargs):
if not self.visualization:
return redirect('homepage')
return super(CreateVisualizationView, self).get(request, *args, **kwargs)
return super().get(request, *args, **kwargs)
def form_valid(self, form):
if not self.visualization:
return redirect('homepage')
form.instance.parameters = self.visualization.to_json()
return super(CreateVisualizationView, self).form_valid(form)
return super().form_valid(form)
class SaveAsVisualizationView(views.AuthorizationMixin, DetailView, CreateView):
@ -132,12 +134,10 @@ class SaveAsVisualizationView(views.AuthorizationMixin, DetailView, CreateView):
def form_valid(self, form):
form.instance.parameters = self.get_object().parameters
return super(SaveAsVisualizationView, self).form_valid(form)
return super().form_valid(form)
def get_initial(self):
return {
'name': '%s %s' % (self.get_object().name, _('(Copy)'))
}
return {'name': '%s %s' % (self.get_object().name, _('(Copy)'))}
class VisualizationView(views.AuthorizationMixin, CubeDisplayMixin, DetailView):
@ -145,7 +145,7 @@ class VisualizationView(views.AuthorizationMixin, CubeDisplayMixin, DetailView):
template_name = 'bijoe/visualization.html'
def get_object(self):
named_visualization = super(VisualizationView, self).get_object()
named_visualization = super().get_object()
if not hasattr(self, 'visualization'):
self.visualization = Visualization.from_json(named_visualization.parameters)
self.cube = self.visualization.cube
@ -165,7 +165,7 @@ class VisualizationView(views.AuthorizationMixin, CubeDisplayMixin, DetailView):
return self.get(request, *args, **kwargs)
def get_context_data(self, **kwargs):
ctx = super(VisualizationView, self).get_context_data(**kwargs)
ctx = super().get_context_data(**kwargs)
initial = {
'representation': self.visualization.representation,
'measure': self.visualization.measure.name,
@ -218,7 +218,7 @@ class VisualizationsView(views.AuthorizationMixin, ListView):
return self.model.all_visualizations()
def get_context_data(self, **kwargs):
ctx = super(VisualizationsView, self).get_context_data(**kwargs)
ctx = super().get_context_data(**kwargs)
ctx['request'] = self.request
return ctx
@ -251,12 +251,14 @@ class VisualizationsJSONView(MultipleObjectMixin, View):
sig = hashlib.sha1(force_bytes(sig)).hexdigest()
path += '?signature=' + sig
data_uri = reverse('visualization-json', kwargs={'pk': visualization.pk})
data.append({
'name': visualization.name,
'slug': visualization.slug,
'path': request.build_absolute_uri(path),
'data-url': request.build_absolute_uri(data_uri),
})
data.append(
{
'name': visualization.name,
'slug': visualization.slug,
'path': request.build_absolute_uri(path),
'data-url': request.build_absolute_uri(data_uri),
}
)
response = HttpResponse(content_type='application/json')
response.write(json.dumps(data))
return response
@ -294,14 +296,16 @@ class VisualizationGeoJSONView(generics.GenericAPIView):
for cell in row.dimensions:
properties[cell.dimension.label] = '%s' % (cell,)
points = row.measures[0].value or []
geojson['features'].append({
'type': 'Feature',
'geometry': {
'type': 'MultiPoint',
'coordinates': [[coord for coord in point] for point in points],
},
'properties': properties,
})
geojson['features'].append(
{
'type': 'Feature',
'geometry': {
'type': 'MultiPoint',
'coordinates': [[coord for coord in point] for point in points],
},
'properties': properties,
}
)
return Response(geojson)
@ -343,29 +347,30 @@ class VisualizationJSONView(generics.GenericAPIView):
elif len(drilldowns) == 0:
data = cell_value(visualization.data()[0].measures[0])
axis = {}
loop.append({
'data': data,
'axis': axis
})
loop.append({'data': data, 'axis': axis})
if not all_visualizations.loop:
data = loop[0]['data']
axis = loop[0]['axis']
else:
axis = loop[0]['axis']
axis['loop'] = [x.label for x in all_visualizations.loop.members(all_visualizations.filters.items())]
axis['loop'] = [
x.label for x in all_visualizations.loop.members(all_visualizations.filters.items())
]
data = [x['data'] for x in loop]
unit = 'seconds' if all_visualizations.measure.type == 'duration' else None
measure = all_visualizations.measure.type
return Response({
'data': data,
'axis': axis,
'format': '1',
'unit': unit, # legacy, prefer measure.
'measure': measure,
})
return Response(
{
'data': data,
'axis': axis,
'format': '1',
'unit': unit, # legacy, prefer measure.
'measure': measure,
}
)
class ExportVisualizationView(views.AuthorizationMixin, DetailView):
@ -387,8 +392,7 @@ class VisualizationsImportView(views.AuthorizationMixin, FormView):
def form_valid(self, form):
try:
visualizations_json = json.loads(
force_text(self.request.FILES['visualizations_json'].read()))
visualizations_json = json.loads(force_text(self.request.FILES['visualizations_json'].read()))
except ValueError:
form.add_error('visualizations_json', _('File is not in the expected JSON format.'))
return self.form_invalid(form)
@ -401,24 +405,31 @@ class VisualizationsImportView(views.AuthorizationMixin, FormView):
if results.get('created') == 0:
message1 = _('No visualization created.')
else:
message1 = ungettext(
'A visualization has been created.',
'%(count)d visualizations have been created.',
results['created']) % {'count': results['created']}
message1 = (
ungettext(
'A visualization has been created.',
'%(count)d visualizations have been created.',
results['created'],
)
% {'count': results['created']}
)
if results.get('updated') == 0:
message2 = _('No visualization updated.')
else:
message2 = ungettext(
'A visualization has been updated.',
'%(count)d visualizations have been updated.',
results['updated']) % {'count': results['updated']}
messages.info(self.request, u'%s %s' % (message1, message2))
message2 = (
ungettext(
'A visualization has been updated.',
'%(count)d visualizations have been updated.',
results['updated'],
)
% {'count': results['updated']}
)
messages.info(self.request, '%s %s' % (message1, message2))
return super(VisualizationsImportView, self).form_valid(form)
return super().form_valid(form)
class VisualizationsExportView(views.AuthorizationMixin, View):
def get(self, request, *args, **kwargs):
response = HttpResponse(content_type='application/json')
response['Content-Disposition'] = (
@ -429,12 +440,12 @@ class VisualizationsExportView(views.AuthorizationMixin, View):
class Select2ChoicesView(View):
def get(self, request, *args, **kwargs):
widget = self.get_widget_or_404()
try:
warehouse = Engine([warehouse for warehouse in get_warehouses()
if warehouse.name == widget.warehouse][0])
warehouse = Engine(
[warehouse for warehouse in get_warehouses() if warehouse.name == widget.warehouse][0]
)
cube = warehouse[widget.cube]
self.dimension = cube.dimensions[widget.dimension]
except IndexError:
@ -448,10 +459,12 @@ class Select2ChoicesView(View):
term = request.GET.get('term', '')
choices = self.get_choices(term, page_number, widget.max_results)
return JsonResponse({
'results': [{'text': label, 'id': s} for s, label in choices],
'more': not(len(choices) < widget.max_results),
})
return JsonResponse(
{
'results': [{'text': label, 'id': s} for s, label in choices],
'more': not (len(choices) < widget.max_results),
}
)
def get_choices(self, term, page_number, max_results):
members = []
@ -460,7 +473,7 @@ class Select2ChoicesView(View):
members.append((None, '__none__', _('None')))
choices = [(s, label) for v, s, label in members if term in label.lower()]
choices = choices[page_number * max_results:(page_number * max_results) + max_results]
choices = choices[page_number * max_results : (page_number * max_results) + max_results]
return choices
def get_widget_or_404(self):

View File

@ -13,13 +13,13 @@ exec(open('/usr/lib/hobo/debian_config_common.py').read())
# SAML2 authentication
AUTHENTICATION_BACKENDS = ('mellon.backends.SAMLBackend',)
MELLON_ATTRIBUTE_MAPPING = {
MELLON_ATTRIBUTE_MAPPING = {
'email': '{attributes[email][0]}',
'first_name': '{attributes[first_name][0]}',
'last_name': '{attributes[last_name][0]}',
}
MELLON_SUPERUSER_MAPPING = {
MELLON_SUPERUSER_MAPPING = {
'is_superuser': 'true',
}

View File

@ -10,7 +10,6 @@ sudo -u bijoe bijoe-manage tenant_command runscript --all-tenants /usr/share/doc
from bijoe.utils import get_warehouses
from bijoe.visualization.models import Visualization
warehouses = get_warehouses()
for visu in Visualization.objects.all():
for warehouse in warehouses:

6
debian/settings.py vendored
View File

@ -14,15 +14,15 @@
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
#ADMINS = (
# ADMINS = (
# # ('User 1', 'watchdog@example.net'),
# # ('User 2', 'janitor@example.net'),
#)
# )
# ALLOWED_HOSTS must be correct in production!
# See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
ALLOWED_HOSTS = [
'*',
'*',
]
# Databases

View File

@ -1,5 +1,3 @@
from __future__ import print_function
import json
import os

View File

@ -1,14 +1,15 @@
#! /usr/bin/env python
import sys
import subprocess
import os
import subprocess
import sys
from distutils.cmd import Command
from distutils.command.build import build as _build
from setuptools import setup, find_packages
from setuptools.command.sdist import sdist
from setuptools import find_packages, setup
from setuptools.command.install_lib import install_lib as _install_lib
from setuptools.command.sdist import sdist
class eo_sdist(sdist):
def run(self):
@ -24,21 +25,21 @@ class eo_sdist(sdist):
def get_version():
'''Use the VERSION, if absent generates a version with git describe, if not
tag exists, take 0.0.0- and add the length of the commit log.
'''
"""Use the VERSION, if absent generates a version with git describe, if not
tag exists, take 0.0.0- and add the length of the commit log.
"""
if os.path.exists('VERSION'):
with open('VERSION', 'r') as v:
with open('VERSION') as v:
return v.read()
if os.path.exists('.git'):
p = subprocess.Popen(['git', 'describe', '--dirty', '--match=v*'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p = subprocess.Popen(
['git', 'describe', '--dirty', '--match=v*'], stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
result = p.communicate()[0]
if p.returncode == 0:
result = result.decode('ascii').split()[0][1:]
else:
result = '0.0.0-%s' % len(subprocess.check_output(
['git', 'rev-list', 'HEAD']).splitlines())
result = '0.0.0-%s' % len(subprocess.check_output(['git', 'rev-list', 'HEAD']).splitlines())
return result.replace('-', '.').replace('.g', '+g')
return '0.0.0'
@ -56,6 +57,7 @@ class compile_translations(Command):
def run(self):
try:
from django.core.management import call_command
for path, dirs, files in os.walk('bijoe'):
if 'locale' not in dirs:
continue
@ -77,27 +79,37 @@ class install_lib(_install_lib):
_install_lib.run(self)
setup(name="bijoe",
version=get_version(),
license="AGPLv3+",
description="BI daashboard from PostgreSQL start schema",
long_description=open('README.rst').read(),
url="http://dev.entrouvert.org/projects/publik-bi/",
author="Entr'ouvert",
author_email="authentic@listes.entrouvert.com",
maintainer="Benjamin Dauvergne",
maintainer_email="bdauvergne@entrouvert.com",
packages=find_packages(),
include_package_data=True,
install_requires=['requests', 'django>=1.11, <2.3', 'psycopg2', 'isodate', 'Django-Select2<6',
'XStatic-ChartNew.js', 'gadjo', 'django-jsonfield<1.3',
'python-dateutil',
'djangorestframework',
'xstatic-select2'],
scripts=['manage.py'],
cmdclass={
'sdist': eo_sdist,
'build': build,
'install_lib': install_lib,
'compile_translations': compile_translations,
})
setup(
name="bijoe",
version=get_version(),
license="AGPLv3+",
description="BI daashboard from PostgreSQL start schema",
long_description=open('README.rst').read(),
url="http://dev.entrouvert.org/projects/publik-bi/",
author="Entr'ouvert",
author_email="authentic@listes.entrouvert.com",
maintainer="Benjamin Dauvergne",
maintainer_email="bdauvergne@entrouvert.com",
packages=find_packages(),
include_package_data=True,
install_requires=[
'requests',
'django>=1.11, <2.3',
'psycopg2',
'isodate',
'Django-Select2<6',
'XStatic-ChartNew.js',
'gadjo',
'django-jsonfield<1.3',
'python-dateutil',
'djangorestframework',
'xstatic-select2',
],
scripts=['manage.py'],
cmdclass={
'sdist': eo_sdist,
'build': build,
'install_lib': install_lib,
'compile_translations': compile_translations,
},
)

View File

@ -1,26 +1,25 @@
import os
import glob
import json
from contextlib import closing, contextmanager
import os
import shutil
import subprocess
import tempfile
import shutil
import pytest
from contextlib import closing, contextmanager
import django_webtest
import psycopg2
from django.db import transaction
import pytest
from django.contrib.auth.models import User
from django.core.management import call_command
from django.db import transaction
def pytest_addoption(parser):
parser.addoption(
'--bijoe-store-table', action='store_true', default=False, help='Store tables value in new_tables.json',
'--bijoe-store-table',
action='store_true',
default=False,
help='Store tables value in new_tables.json',
)
@ -44,14 +43,14 @@ def john_doe(db):
@pytest.fixture
def admin(db):
u = User(username='super.user', first_name='Super', last_name='User',
email='super.user@example.net')
u = User(username='super.user', first_name='Super', last_name='User', email='super.user@example.net')
u.set_password('super.user')
u.is_superuser = True
u.is_staff = True
u.save()
return u
SCHEMA_PATHS = os.path.join(os.path.dirname(__file__), 'fixtures/')
@ -83,9 +82,19 @@ def load_schema_db(schema):
# load data
for sql_path in sorted(glob.glob(os.path.join(schema_dir, '*.sql'))):
process = subprocess.Popen(['psql', '-c', '\\set ON_ERROR_STOP on', '--single-transaction', database_name, '-f', sql_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
process = subprocess.Popen(
[
'psql',
'-c',
'\\set ON_ERROR_STOP on',
'--single-transaction',
database_name,
'-f',
sql_path,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
return_code = process.returncode
assert return_code == 0, [stdout, stderr]
@ -97,7 +106,7 @@ def load_schema_db(schema):
'schema_dir': schema_dir,
'database_name': database_name,
'bijoe_schemas': [os.path.join(bijoe_schema_dir, '*_schema.json')],
'fixtures': fixtures
'fixtures': fixtures,
}
tables_path = os.path.join(schema_dir, 'tables.json')
if os.path.exists(tables_path):

View File

@ -1,65 +1,71 @@
# -*- coding: utf-8 -*-
from bijoe.schemas import Warehouse
def test_simple_parsing():
Warehouse.from_json({
'name': 'coin',
'label': 'coin',
'pg_dsn': 'dbname=zozo',
'search_path': ['cam', 'public'],
'cubes': [
{
'name': 'all_formdata',
'label': 'Tous les formulaires',
'fact_table': 'formdata',
'key': 'id',
'joins': [
{
'name': 'formdef',
'master': '{fact_table}.formdef_id',
'table': 'formdef',
'detail': 'formdef.id',
}
],
'dimensions': [
{
'label': 'formulaire',
'name': 'formdef',
'type': 'integer',
'join': ['formdef'],
'value': 'formdef.id',
'value_label': 'formdef.label',
'order_by': 'formdef.label'
},
{
'name': 'receipt_time',
'label': 'date de soumission',
'join': ['receipt_time'],
'type': 'date',
'value': 'receipt_time.date'
}
],
'measures': [
{
'type': 'integer',
'label': 'Nombre de demandes',
'expression': 'count({fact_table}.id)',
'name': 'count'
},
{
'type': 'integer',
'label': u'Délai de traitement',
'expression': 'avg((to_char(endpoint_delay, \'9999.999\') || \' days\')::interval)',
'name': 'avg_endpoint_delay'
},
{
'type': 'percent',
'label': 'Pourcentage',
'expression': 'count({fact_table}.id) * 100. / (select count({fact_table}.id) from {table_expression} where {where_conditions})',
'name': 'percentage'
}
]
}
],
})
Warehouse.from_json(
{
'name': 'coin',
'label': 'coin',
'pg_dsn': 'dbname=zozo',
'search_path': ['cam', 'public'],
'cubes': [
{
'name': 'all_formdata',
'label': 'Tous les formulaires',
'fact_table': 'formdata',
'key': 'id',
'joins': [
{
'name': 'formdef',
'master': '{fact_table}.formdef_id',
'table': 'formdef',
'detail': 'formdef.id',
}
],
'dimensions': [
{
'label': 'formulaire',
'name': 'formdef',
'type': 'integer',
'join': ['formdef'],
'value': 'formdef.id',
'value_label': 'formdef.label',
'order_by': 'formdef.label',
},
{
'name': 'receipt_time',
'label': 'date de soumission',
'join': ['receipt_time'],
'type': 'date',
'value': 'receipt_time.date',
},
],
'measures': [
{
'type': 'integer',
'label': 'Nombre de demandes',
'expression': 'count({fact_table}.id)',
'name': 'count',
},
{
'type': 'integer',
'label': 'Délai de traitement',
'expression': (
'avg((to_char(endpoint_delay, \'9999.999\') || \' days\')::interval)'
),
'name': 'avg_endpoint_delay',
},
{
'type': 'percent',
'label': 'Pourcentage',
'expression': (
'count({fact_table}.id) * 100. / (select count({fact_table}.id) from'
' {table_expression} where {where_conditions})'
),
'name': 'percentage',
},
],
}
],
}
)

View File

@ -20,9 +20,13 @@ from bijoe.hobo_agent.management.commands import hobo_deploy
def test_schema_from_url():
for hash_length in [4, 5, 6, 7]:
for length in [64, 65, 66]:
assert len(hobo_deploy.schema_from_url('https://' + ('x' * length), hash_length=hash_length)) == 63
assert (
len(hobo_deploy.schema_from_url('https://' + ('x' * length), hash_length=hash_length)) == 63
)
schema = hobo_deploy.schema_from_url('https://demarches-saint-didier-au-mont-dor.guichet-recette.grandlyon.com/')
schema = hobo_deploy.schema_from_url(
'https://demarches-saint-didier-au-mont-dor.guichet-recette.grandlyon.com/'
)
assert len(schema) == 63
assert schema == 'demarches_saint_didier_au_mo0757cfguichet_recette_grandlyon_com'

View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# bijoe - BI dashboard
# Copyright (C) 2015 Entr'ouvert
#
@ -17,13 +16,10 @@
from contextlib import contextmanager
from psycopg2.extensions import parse_dsn
import pytest
from django.utils.six.moves import configparser as ConfigParser
import sentry_sdk
from django.utils.six.moves import configparser as ConfigParser
from psycopg2.extensions import parse_dsn
from bijoe.hobo_agent.management.commands import hobo_deploy
@ -33,7 +29,7 @@ def donothing(tenant):
yield
class FakeTenant(object):
class FakeTenant:
domain_url = 'fake.tenant.com'
def __init__(self, directory):
@ -45,9 +41,7 @@ class FakeTenant(object):
@pytest.fixture
def sentry():
sentry_sdk.init(
dsn='https://1234@sentry.example.com/1',
environment='prod')
sentry_sdk.init(dsn='https://1234@sentry.example.com/1', environment='prod')
yield
sentry_sdk.init()

View File

@ -1,7 +1,3 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import os
import shutil
@ -13,8 +9,8 @@ from django.core.management import call_command
from django.utils.encoding import force_bytes
from django.utils.six import StringIO
from bijoe.visualization.models import Visualization
from bijoe.utils import import_site
from bijoe.visualization.models import Visualization
pytestmark = pytest.mark.django_db
@ -35,7 +31,7 @@ def test_import_export(schema1, app):
'representation': 'table',
'loop': '',
'filters': {},
'drilldown_x': 'date__yearmonth'
'drilldown_x': 'date__yearmonth',
}
def create_visu(i=0):

View File

@ -1,16 +1,16 @@
# -*- coding: utf-8 -*-
from datetime import date
from bijoe.relative_time import RelativeDate
def test_relative_date():
today = date(2016, 3, 3)
assert RelativeDate(u'cette année', today=today) == date(2016, 1, 1)
assert RelativeDate(u'ce mois', today=today) == date(2016, 3, 1)
assert RelativeDate(u'le mois dernier', today=today) == date(2016, 2, 1)
assert RelativeDate(u'les 4 derniers mois', today=today) == date(2015, 11, 1)
assert RelativeDate(u'le mois prochain', today=today) == date(2016, 4, 1)
assert RelativeDate(u'les 3 prochains mois', today=today) == date(2016, 6, 1)
assert RelativeDate(u' cette semaine', today=today) == date(2016, 2, 29)
assert RelativeDate(u' maintenant', today=today) == today
assert RelativeDate(u'2016-01-01', today=today) == date(2016, 1, 1)
assert RelativeDate('cette année', today=today) == date(2016, 1, 1)
assert RelativeDate('ce mois', today=today) == date(2016, 3, 1)
assert RelativeDate('le mois dernier', today=today) == date(2016, 2, 1)
assert RelativeDate('les 4 derniers mois', today=today) == date(2015, 11, 1)
assert RelativeDate('le mois prochain', today=today) == date(2016, 4, 1)
assert RelativeDate('les 3 prochains mois', today=today) == date(2016, 6, 1)
assert RelativeDate(' cette semaine', today=today) == date(2016, 2, 29)
assert RelativeDate(' maintenant', today=today) == today
assert RelativeDate('2016-01-01', today=today) == date(2016, 1, 1)

View File

@ -1,11 +1,9 @@
# -*- coding: utf-8 -*-
import json
from utils import login, get_table, get_ods_table, get_ods_document, request_select2
from utils import get_ods_document, get_ods_table, get_table, login, request_select2
from bijoe.visualization.ods import OFFICE_NS, TABLE_NS
from bijoe.visualization.models import Visualization as VisualizationModel
from bijoe.visualization.ods import OFFICE_NS, TABLE_NS
from bijoe.visualization.utils import Visualization
@ -15,7 +13,7 @@ def test_simple(schema1, app, admin):
response = response.click('schema1')
response = response.click('Facts 1')
assert 'big-msg-info' in response
assert u'le champ « pouët »' in response
assert 'le champ « pouët »' in response
assert 'warning2' in response
form = response.form
form.set('representation', 'table')
@ -24,9 +22,21 @@ def test_simple(schema1, app, admin):
response = form.submit('visualize')
assert 'big-msg-info' not in response
assert get_table(response) == [
['Inner SubCategory', u'sub\xe910', u'sub\xe911', u'sub\xe94', u'sub\xe95', u'sub\xe96', u'sub\xe98',
u'sub\xe99', u'sub\xe97', u'sub\xe92', u'sub\xe93', u'sub\xe91'],
['number of rows', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '15']
[
'Inner SubCategory',
'sub\xe910',
'sub\xe911',
'sub\xe94',
'sub\xe95',
'sub\xe96',
'sub\xe98',
'sub\xe99',
'sub\xe97',
'sub\xe92',
'sub\xe93',
'sub\xe91',
],
['number of rows', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '15'],
]
form = response.form
form.set('representation', 'table')
@ -35,7 +45,7 @@ def test_simple(schema1, app, admin):
response = form.submit('visualize')
assert 'big-msg-info' not in response
assert get_table(response) == [
['mois (Date)', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt'],
['mois (Date)', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt'],
['number of rows', '10', '1', '1', '1', '1', '1', '1', '1'],
]
@ -54,14 +64,14 @@ def test_truncated_previous_year_range(schema1, app, admin, freezer):
freezer.move_to('2019-01-01 01:00:00')
response = form.submit('visualize')
assert get_table(response) == [
['', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt', 'Total'],
['', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt', 'Total'],
['2017', '0', '0', '0', '0', '0', '0', '0', '0', '0'],
]
freezer.move_to('2018-01-01 01:00:00')
response = form.submit('visualize')
assert get_table(response) == [
['', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt', 'Total'],
['', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt', 'Total'],
['2017', '10', '1', '1', '1', '1', '1', '1', '1', '17'],
]
@ -77,7 +87,9 @@ def test_boolean_dimension(schema1, app, admin):
form.set('drilldown_x', 'boolean')
response = form.submit('visualize')
assert get_table(response) == [['Boolean', 'Oui', 'Non'], ['number of rows', '8', '9']]
form['filter__boolean'].force_value([o[0] for o in form.fields['filter__boolean'][0].options if o[2] == 'Oui'][0])
form['filter__boolean'].force_value(
[o[0] for o in form.fields['filter__boolean'][0].options if o[2] == 'Oui'][0]
)
response = form.submit('visualize')
assert get_table(response) == [['Boolean', 'Oui', 'Non'], ['number of rows', '8', '0']]
@ -92,7 +104,10 @@ def test_string_dimension(schema1, app, admin):
form.set('measure', 'simple_count')
form.set('drilldown_x', 'string')
response = form.submit('visualize')
assert get_table(response) == [['String', 'a', 'b', 'c', 'Aucun(e)'], ['number of rows', '11', '2', '3', '1']]
assert get_table(response) == [
['String', 'a', 'b', 'c', 'Aucun(e)'],
['number of rows', '11', '2', '3', '1'],
]
form['filter__string'].force_value(['a', 'b', '__none__'])
response = form.submit('visualize')
assert get_table(response) == [['String', 'a', 'b', 'Aucun(e)'], ['number of rows', '11', '2', '1']]
@ -100,18 +115,20 @@ def test_string_dimension(schema1, app, admin):
def test_string_dimension_json_data(schema1, app, admin):
# test conversion to Javascript declaration
visu = Visualization.from_json({
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'table',
'measure': 'simple_count',
'drilldown_x': 'string'
})
visu = Visualization.from_json(
{
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'table',
'measure': 'simple_count',
'drilldown_x': 'string',
}
)
assert json.loads(json.dumps(visu.json_data())) == [
{u'coords': [{u'value': u'a'}], u'measures': [{u'value': 11}]},
{u'coords': [{u'value': u'b'}], u'measures': [{u'value': 2}]},
{u'coords': [{u'value': u'c'}], u'measures': [{u'value': 3}]},
{u'coords': [{u'value': u'Aucun(e)'}], u'measures': [{u'value': 1}]}
{'coords': [{'value': 'a'}], 'measures': [{'value': 11}]},
{'coords': [{'value': 'b'}], 'measures': [{'value': 2}]},
{'coords': [{'value': 'c'}], 'measures': [{'value': 3}]},
{'coords': [{'value': 'Aucun(e)'}], 'measures': [{'value': 1}]},
]
@ -126,16 +143,26 @@ def test_item_dimension(schema1, app, admin):
form.set('drilldown_x', 'outersubcategory')
response = form.submit('visualize')
assert get_table(response) == [
['Outer SubCategory', u'sub\xe910', u'sub\xe911', u'sub\xe94', u'sub\xe95', u'sub\xe96', u'sub\xe98',
u'sub\xe99', u'sub\xe97', u'sub\xe92', u'sub\xe93', u'sub\xe91', 'Aucun(e)'],
['number of rows', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '15', '1']
[
'Outer SubCategory',
'sub\xe910',
'sub\xe911',
'sub\xe94',
'sub\xe95',
'sub\xe96',
'sub\xe98',
'sub\xe99',
'sub\xe97',
'sub\xe92',
'sub\xe93',
'sub\xe91',
'Aucun(e)',
],
['number of rows', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1', '15', '1'],
]
form['filter__outersubcategory'].force_value(['__none__'])
response = form.submit('visualize')
assert get_table(response) == [
['Outer SubCategory', 'Aucun(e)'],
['number of rows', '1']
]
assert get_table(response) == [['Outer SubCategory', 'Aucun(e)'], ['number of rows', '1']]
def test_yearmonth_drilldown(schema1, app, admin):
@ -149,9 +176,18 @@ def test_yearmonth_drilldown(schema1, app, admin):
form.set('drilldown_x', 'date__yearmonth')
response = form.submit('visualize')
assert get_table(response) == [
[u'ann\xe9e et mois (Date)', '01/2017', '02/2017', '03/2017',
'04/2017', '05/2017', '06/2017', '07/2017', '08/2017'],
['number of rows', '10', '1', '1', '1', '1', '1', '1', '1']
[
'ann\xe9e et mois (Date)',
'01/2017',
'02/2017',
'03/2017',
'04/2017',
'05/2017',
'06/2017',
'07/2017',
'08/2017',
],
['number of rows', '10', '1', '1', '1', '1', '1', '1', '1'],
]
@ -196,53 +232,47 @@ def test_truncated_previous_year_range_on_datetime(schema1, app, admin, freezer)
freezer.move_to('2019-01-01 01:00:00')
response = form.submit('visualize')
assert get_table(response) == [
['', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt', 'Total'],
['', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt', 'Total'],
['2017', '0', '0', '0', '0', '0', '0', '0', '0', '0'],
]
freezer.move_to('2018-01-01 01:00:00')
response = form.submit('visualize')
assert get_table(response) == [
['', 'janvier', u'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', u'ao\xfbt', 'Total'],
['', 'janvier', 'f\xe9vrier', 'mars', 'avril', 'mai', 'juin', 'juillet', 'ao\xfbt', 'Total'],
['2017', '10', '1', '1', '1', '1', '1', '1', '1', '17'],
]
def test_none_percent_json_data_0d(schema1, app, admin):
# test conversion to Javascript declaration
visu = Visualization.from_json({
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'percent',
})
assert visu.json_data() == [{u'coords': [], u'measures': [{u'value': 100.0}]}]
visu = Visualization.from_json(
{
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'percent',
}
)
assert visu.json_data() == [{'coords': [], 'measures': [{'value': 100.0}]}]
def test_none_percent_json_data_2d(schema1, app, admin):
# test conversion to Javascript declaration
visu = Visualization.from_json({
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'percent',
'drilldown_y': 'leftcategory',
'drilldown_x': 'date__year',
})
assert visu.json_data() == [
visu = Visualization.from_json(
{
'coords': [{'value': u'2017'}, {'value': u'cat\xe92'}],
'measures': [{'value': 0}]
},
{
'coords': [{'value': u'2017'}, {'value': u'cat\xe93'}],
'measures': [{'value': 0}]},
{
'coords': [{'value': u'2017'}, {'value': u'cat\xe91'}],
'measures': [{'value': 94.11764705882354}]},
{
'coords': [{'value': u'2017'}, {'value': u'Aucun(e)'}],
'measures': [{'value': 5.882352941176471}]
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'percent',
'drilldown_y': 'leftcategory',
'drilldown_x': 'date__year',
}
)
assert visu.json_data() == [
{'coords': [{'value': '2017'}, {'value': 'cat\xe92'}], 'measures': [{'value': 0}]},
{'coords': [{'value': '2017'}, {'value': 'cat\xe93'}], 'measures': [{'value': 0}]},
{'coords': [{'value': '2017'}, {'value': 'cat\xe91'}], 'measures': [{'value': 94.11764705882354}]},
{'coords': [{'value': '2017'}, {'value': 'Aucun(e)'}], 'measures': [{'value': 5.882352941176471}]},
]
@ -258,96 +288,89 @@ def test_geoloc(schema1, app, admin):
'measure': 'percent',
'drilldown_y': 'outercategory',
'drilldown_x': 'date__year',
})
},
)
response = app.get('/visualization/%d/geojson/' % visu.pk)
assert response.json == {
'type': 'FeatureCollection',
'features': [{
u'geometry': {
u'coordinates': [
[1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0],
[1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0],
[1.0, 1.0], [1.0, 1.0], [1.0, 1.0], [1.0, 1.0]
],
u'type': u'MultiPoint'
'features': [
{
'geometry': {
'coordinates': [
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
[1.0, 1.0],
],
'type': 'MultiPoint',
},
'properties': {'Outer Category': 'cat\xe91', 'ann\xe9e (Date)': '2017'},
'type': 'Feature',
},
u'properties': {
u'Outer Category': u'cat\xe91',
u'ann\xe9e (Date)': u'2017'
{
'geometry': {'coordinates': [[1.0, 1.0]], 'type': 'MultiPoint'},
'properties': {'Outer Category': 'Aucun(e)', 'ann\xe9e (Date)': '2017'},
'type': 'Feature',
},
u'type': u'Feature'
},
{
u'geometry': {
u'coordinates': [[1.0, 1.0]],
u'type': u'MultiPoint'
{
'geometry': {'coordinates': [], 'type': 'MultiPoint'},
'properties': {'Outer Category': 'cat\xe92', 'ann\xe9e (Date)': 'Aucun(e)'},
'type': 'Feature',
},
u'properties': {
u'Outer Category': u'Aucun(e)',
u'ann\xe9e (Date)': u'2017'
{
'geometry': {'coordinates': [], 'type': 'MultiPoint'},
'properties': {'Outer Category': 'cat\xe93', 'ann\xe9e (Date)': 'Aucun(e)'},
'type': 'Feature',
},
u'type': u'Feature'
},
{
u'geometry': {
u'coordinates': [],
u'type': u'MultiPoint'
{
'geometry': {'coordinates': [], 'type': 'MultiPoint'},
'properties': {'Outer Category': 'cat\xe91', 'ann\xe9e (Date)': 'Aucun(e)'},
'type': 'Feature',
},
u'properties': {
u'Outer Category': u'cat\xe92',
u'ann\xe9e (Date)': u'Aucun(e)'
},
u'type': u'Feature'
},
{
u'geometry': {
u'coordinates': [],
u'type': u'MultiPoint'
},
u'properties': {
u'Outer Category': u'cat\xe93',
u'ann\xe9e (Date)': u'Aucun(e)'
},
u'type': u'Feature'
},
{
u'geometry': {
u'coordinates': [],
u'type': u'MultiPoint'
},
u'properties': {
u'Outer Category': u'cat\xe91',
u'ann\xe9e (Date)': u'Aucun(e)'
},
u'type': u'Feature'
}
]}
],
}
def test_filter_type_mismatch(schema1, app, admin):
# test conversion to Javascript declaration
visu = Visualization.from_json({
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'simple_count',
'filters': {
'string': [1],
visu = Visualization.from_json(
{
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'simple_count',
'filters': {
'string': [1],
},
}
})
)
assert visu.json_data() == [{'coords': [], 'measures': [{'value': 0}]}]
def test_empty_filter(schema1, app, admin):
visu = Visualization.from_json({
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'simple_count',
'filters': {
'innercategory': [],
visu = Visualization.from_json(
{
'warehouse': 'schema1',
'cube': 'facts1',
'representation': 'graphical',
'measure': 'simple_count',
'filters': {
'innercategory': [],
},
}
})
)
assert visu.json_data() == [{'coords': [], 'measures': [{'value': 17}]}]
@ -362,10 +385,7 @@ def test_json_dimensions(schema1, app, admin):
form.set('measure', 'simple_count')
form.set('drilldown_x', 'a')
response = form.submit('visualize')
assert get_table(response) == [
['A', 'x', 'y', 'z'],
['number of rows', '7', '9', '1']
]
assert get_table(response) == [['A', 'x', 'y', 'z'], ['number of rows', '7', '9', '1']]
assert 'filter__a' in form.fields
choices = [o['id'] for o in request_select2(app, response, 'filter__a')['results']]
@ -373,10 +393,7 @@ def test_json_dimensions(schema1, app, admin):
form['filter__a'].force_value(['x', 'y'])
response = form.submit('visualize')
assert get_table(response) == [
['A', 'x', 'y', 'z'],
['number of rows', '7', '9', '0']
]
assert get_table(response) == [['A', 'x', 'y', 'z'], ['number of rows', '7', '9', '0']]
def test_json_dimensions_having_percent(schema1, app, admin):
@ -392,7 +409,7 @@ def test_json_dimensions_having_percent(schema1, app, admin):
response = form.submit('visualize')
assert get_table(response) == [
['A', 'x', 'y', 'z'],
['pourcentage des demandes', '41,18 %', '52,94 %', '5,88 %']
['pourcentage des demandes', '41,18 %', '52,94 %', '5,88 %'],
]
assert 'filter__a' in form.fields
@ -403,7 +420,7 @@ def test_json_dimensions_having_percent(schema1, app, admin):
response = form.submit('visualize')
assert get_table(response) == [
['A', 'x', 'y', 'z'],
['pourcentage des demandes', '43,75 %', '56,25 %', '0,00 %']
['pourcentage des demandes', '43,75 %', '56,25 %', '0,00 %'],
]

View File

@ -4,20 +4,14 @@ import re
import pytest
from tabulate import tabulate
from utils import login, get_table
from utils import get_table, login
def pytest_generate_tests(metafunc):
if hasattr(metafunc, 'function'):
fcode = metafunc.function.__code__
if 'visualization' in fcode.co_varnames[:fcode.co_argcount]:
with open(
os.path.join(
os.path.dirname(__file__),
'fixtures',
'schema2',
'tables.json')) as fd:
if 'visualization' in fcode.co_varnames[: fcode.co_argcount]:
with open(os.path.join(os.path.dirname(__file__), 'fixtures', 'schema2', 'tables.json')) as fd:
tables = json.load(fd)
metafunc.parametrize(['visualization'], [[x] for x in tables])
@ -53,6 +47,4 @@ def test_simple(request, schema2, app, admin, visualization):
d[visualization] = table
with open(new_table_path, 'w') as fd:
json.dump(d, fd, indent=4, sort_keys=True, separators=(',', ': '))
assert_equal_tables(
schema2['tables'][visualization],
table)
assert_equal_tables(schema2['tables'][visualization], table)

View File

@ -15,24 +15,22 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import pytest
from django.utils.translation import ugettext as _
from bijoe.schemas import (
Dimension,
)
from bijoe.schemas import Dimension
def test_absent_label():
assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'integer'}).absent_label == _('None')
assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'string'}).absent_label == _('None')
assert Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'bool'}).absent_label == _('N/A')
assert Dimension.from_json(
{'name': 'x', 'value': 'x', 'type': 'boolean', 'absent_label': 'coin'}).absent_label == 'coin'
assert (
Dimension.from_json(
{'name': 'x', 'value': 'x', 'type': 'boolean', 'absent_label': 'coin'}
).absent_label
== 'coin'
)
with pytest.raises(NotImplementedError):
Dimension.from_json({'name': 'x', 'value': 'x', 'type': 'coin'}).absent_label

View File

@ -38,7 +38,7 @@ def test_signature():
assert not signature.check_string(STRING, signature.sign_string(STRING, KEY), OTHER_KEY)
assert not signature.check_query(signature.sign_query(QUERY, KEY), OTHER_KEY)
assert not signature.check_url(signature.sign_url(URL, KEY), OTHER_KEY)
#assert not signature.check_url('%s&foo=bar' % signature.sign_url(URL, KEY), KEY)
# assert not signature.check_url('%s&foo=bar' % signature.sign_url(URL, KEY), KEY)
# Test URL is preserved
assert URL in signature.sign_url(URL, KEY)
@ -50,28 +50,29 @@ def test_signature():
assert '&nonce=' in signature.sign_url(URL, KEY)
# Test unicode key conversion to UTF-8
assert signature.check_url(signature.sign_url(URL, u'\xe9\xe9'), b'\xc3\xa9\xc3\xa9')
assert signature.check_url(signature.sign_url(URL, b'\xc3\xa9\xc3\xa9'), u'\xe9\xe9')
assert signature.check_url(signature.sign_url(URL, '\xe9\xe9'), b'\xc3\xa9\xc3\xa9')
assert signature.check_url(signature.sign_url(URL, b'\xc3\xa9\xc3\xa9'), '\xe9\xe9')
# Test timedelta parameter
now = datetime.datetime.utcnow()
assert '&timestamp=%s' % urllib.quote(now.strftime('%Y-%m-%dT%H:%M:%SZ')) in \
signature.sign_url(URL, KEY, timestamp=now)
assert '&timestamp=%s' % urllib.quote(now.strftime('%Y-%m-%dT%H:%M:%SZ')) in signature.sign_url(
URL, KEY, timestamp=now
)
# Test nonce parameter
assert '&nonce=uuu&' in signature.sign_url(URL, KEY, nonce='uuu')
# Test known_nonce
assert signature.check_url(signature.sign_url(URL, KEY), KEY,
known_nonce=lambda nonce: nonce == 'xxx')
assert signature.check_url(signature.sign_url(URL, KEY, nonce='xxx'), KEY,
known_nonce=lambda nonce: nonce == 'xxx')
assert signature.check_url(signature.sign_url(URL, KEY), KEY, known_nonce=lambda nonce: nonce == 'xxx')
assert signature.check_url(
signature.sign_url(URL, KEY, nonce='xxx'), KEY, known_nonce=lambda nonce: nonce == 'xxx'
)
# Test timedelta
now = (datetime.datetime.utcnow() - datetime.timedelta(seconds=29))
now = datetime.datetime.utcnow() - datetime.timedelta(seconds=29)
assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY)
now = (datetime.datetime.utcnow() - datetime.timedelta(seconds=30))
now = datetime.datetime.utcnow() - datetime.timedelta(seconds=30)
assert not signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY)
now = (datetime.datetime.utcnow() - datetime.timedelta(seconds=2))
now = datetime.datetime.utcnow() - datetime.timedelta(seconds=2)
assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY)
assert signature.check_url(signature.sign_url(URL, KEY, timestamp=now), KEY, timedelta=2)

View File

@ -17,19 +17,17 @@
import copy
import hashlib
import json
from unittest import mock
import mock
import pytest
from webtest import Upload
from django.urls import reverse
from django.utils.encoding import force_bytes
from utils import login
from webtest import Upload
from bijoe.visualization.models import Visualization
from bijoe.visualization.signature import sign_url
from utils import login
@pytest.fixture
def visualization():
@ -42,7 +40,9 @@ def visualization():
'representation': 'table',
'loop': '',
'filters': {},
'drilldown_x': 'date__yearmonth'})
'drilldown_x': 'date__yearmonth',
},
)
def test_simple_user_403(app, john_doe):
@ -78,11 +78,13 @@ def test_visualizations_json_api(schema1, app, admin, settings):
'default': {
'verif_orig': orig,
'secret': key,
}}}
}
}
}
url = '%s?orig=%s' % (reverse('visualizations-json'), orig)
url = sign_url(url, key)
resp = app.get(url, status=200)
assert set([x['slug'] for x in resp.json]) == set(['test', 'test-2', 'test-3', 'test-4'])
assert {x['slug'] for x in resp.json} == {'test', 'test-2', 'test-3', 'test-4'}
url = '%s?orig=%s' % (reverse('visualizations-json'), orig)
url = sign_url(url, 'wrong-key')
@ -97,7 +99,7 @@ def test_visualizations_json_api(schema1, app, admin, settings):
login(app, admin)
resp = app.get(reverse('visualizations-json'))
assert set([x['slug'] for x in resp.json]) == set(['test', 'test-2', 'test-3', 'test-4'])
assert {x['slug'] for x in resp.json} == {'test', 'test-2', 'test-3', 'test-4'}
def test_visualization_json_api(schema1, app, admin, visualization):
@ -105,7 +107,18 @@ def test_visualization_json_api(schema1, app, admin, visualization):
resp = app.get(reverse('visualization-json', kwargs={'pk': visualization.id}))
# values from test_schem1/test_yearmonth_drilldown
assert resp.json == {
'axis': {'x_labels': ['01/2017', '02/2017', '03/2017', '04/2017', '05/2017', '06/2017', '07/2017', '08/2017']},
'axis': {
'x_labels': [
'01/2017',
'02/2017',
'03/2017',
'04/2017',
'05/2017',
'06/2017',
'07/2017',
'08/2017',
]
},
'data': [10, 1, 1, 1, 1, 1, 1, 1],
'format': '1',
'unit': None,
@ -121,9 +134,28 @@ def test_visualization_json_api_duration(schema1, app, admin, visualization):
resp = app.get(reverse('visualization-json', kwargs={'pk': visualization.id}))
# values from test_schem1/test_yearmonth_drilldown
assert resp.json == {
'axis': {'x_labels': ['01/2017', '02/2017', '03/2017', '04/2017', '05/2017', '06/2017', '07/2017', '08/2017']},
'data': [536968800.0, 539258400.0, 541677600.0, 544352400.0,
546944400.0, 549622800.0, 552214800.0, 554893200.0],
'axis': {
'x_labels': [
'01/2017',
'02/2017',
'03/2017',
'04/2017',
'05/2017',
'06/2017',
'07/2017',
'08/2017',
]
},
'data': [
536968800.0,
539258400.0,
541677600.0,
544352400.0,
546944400.0,
549622800.0,
552214800.0,
554893200.0,
],
'format': '1',
'unit': 'seconds',
'measure': 'duration',
@ -194,7 +226,9 @@ def test_import_visualization(schema1, app, admin, visualization, settings, free
# existing visualization
resp = app.get('/', status=200)
resp = resp.click('Import')
resp.form['visualizations_json'] = Upload('export.json', visualization_export.encode('utf-8'), 'application/json')
resp.form['visualizations_json'] = Upload(
'export.json', visualization_export.encode('utf-8'), 'application/json'
)
resp = resp.form.submit().follow()
assert 'No visualization created. A visualization has been updated.' in resp.text
assert Visualization.objects.count() == 1
@ -203,7 +237,9 @@ def test_import_visualization(schema1, app, admin, visualization, settings, free
Visualization.objects.all().delete()
resp = app.get('/')
resp = resp.click('Import')
resp.form['visualizations_json'] = Upload('export.json', visualization_export.encode('utf-8'), 'application/json')
resp.form['visualizations_json'] = Upload(
'export.json', visualization_export.encode('utf-8'), 'application/json'
)
resp = resp.form.submit().follow()
assert 'A visualization has been created. No visualization updated.' in resp.text
assert Visualization.objects.count() == 1
@ -220,8 +256,8 @@ def test_import_visualization(schema1, app, admin, visualization, settings, free
resp = app.get('/', status=200)
resp = resp.click('Import')
resp.form['visualizations_json'] = Upload(
'export.json',
json.dumps(visualizations).encode('utf-8'), 'application/json')
'export.json', json.dumps(visualizations).encode('utf-8'), 'application/json'
)
resp = resp.form.submit().follow()
assert '2 visualizations have been created. A visualization has been updated.' in resp.text
assert Visualization.objects.count() == 3
@ -235,7 +271,9 @@ def test_import_visualization(schema1, app, admin, visualization, settings, free
resp = app.get('/')
resp = resp.click('Import')
resp.form['visualizations_json'] = Upload('export.json', visualizations_export.encode('utf-8'), 'application/json')
resp.form['visualizations_json'] = Upload(
'export.json', visualizations_export.encode('utf-8'), 'application/json'
)
resp = resp.form.submit().follow()
assert '3 visualizations have been created. No visualization updated.' in resp.text
assert Visualization.objects.count() == 3

View File

@ -1,6 +1,6 @@
import io
import zipfile
import xml.etree.ElementTree as ET
import zipfile
from django.conf import settings
@ -35,8 +35,8 @@ def get_table(response):
def xml_node_text_content(node):
'''Extract text content from node and all its children. Equivalent to
xmlNodeGetContent from libxml.'''
"""Extract text content from node and all its children. Equivalent to
xmlNodeGetContent from libxml."""
if node is None:
return ''
@ -50,7 +50,8 @@ def xml_node_text_content(node):
if child.tail:
s.append(child.tail)
return s
return u''.join(helper(node))
return ''.join(helper(node))
def get_ods_document(response):