tests: wip add tests with bijoe

Jenkinsfile

@@ -2,10 +2,19 @@
 
 pipeline {
     agent any
+    options { disableConcurrentBuilds() }
+    environment {
+        TMPDIR = "/tmp/$BUILD_TAG"
+    }
     stages {
         stage('Unit Tests') {
             steps {
-                sh 'tox -rv'
+                sh """
+mkdir ${env.TMPDIR}
+virtualenv venv
+./venv/bin/pip install tox
+PGPORT=`python -c 'import struct; import socket; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` pg_virtualenv -o fsync=off ./venv/bin/tox -rv
+"""
             }
             post {
                 always {
@@ -35,7 +44,8 @@ pipeline {
                 utils.mail_notify(currentBuild, env, 'admin+jenkins-wcs-olap@entrouvert.com')
             }
         }
-        success {
+        cleanup {
+            sh "rm -rf ${env.TMPDIR}"
             cleanWs()
         }
     }

tests/bijoe_settings.py

@@ -0,0 +1 @@
+BIJOE_CACHE = False

tests/conftest.py

@@ -11,6 +11,7 @@ from collections import namedtuple
 
 import psycopg2
 import pytest
+import django_webtest
 
 import utils
 
@@ -22,7 +23,18 @@ class Database(object):
     def __init__(self):
         self.db_name = 'db%s' % random.getrandbits(20)
         self.dsn = 'dbname=%s' % self.db_name
-        with closing(psycopg2.connect('')) as conn:
+        self.connect_kwargs = {
+            'dbname': 'postgres'
+        }
+        for variable, key in [
+                ('PGHOST', 'host'),
+                ('PGUSER', 'user'),
+                ('PGPASSWORD', 'password'),
+                ('PGPORT', 'port'),
+                ('PGDATABASE', 'dbname')]:
+            if variable in os.environ:
+                self.connect_kwargs[key] = os.environ[variable]
+        with closing(psycopg2.connect(**self.connect_kwargs)) as conn:
             conn.set_isolation_level(0)
             with conn.cursor() as cursor:
                 cursor.execute('CREATE DATABASE %s' % self.db_name)
@@ -31,7 +43,7 @@ class Database(object):
         return closing(psycopg2.connect(self.dsn))
 
     def delete(self):
-        with closing(psycopg2.connect('')) as conn:
+        with closing(psycopg2.connect(**self.connect_kwargs)) as conn:
             conn.set_isolation_level(0)
             with conn.cursor() as cursor:
                 cursor.execute('DROP DATABASE IF EXISTS %s' % self.db_name)
@@ -239,3 +251,30 @@ schema = olap
             sys.argv = old_argv
     f.model_dir = model_dir
     return f
+
+
+@pytest.fixture
+def admin(db):
+    from django.contrib.auth.models import User
+
+    u = User(username='admin', first_name='A', last_name='Dmin',
+             email='super.user@example.net')
+    u.set_password('admin')
+    u.is_superuser = True
+    u.is_staff = True
+    u.save()
+    return u
+
+
+@pytest.fixture
+def bijoe(db, settings, olap_cmd):
+    olap_cmd()
+    settings.BIJOE_SCHEMAS = [str(olap_cmd.model_dir) + '/*.model']
+    settings.TEMPLATE_DEBUG = True
+    settings.DEBUG = True
+    wtm = django_webtest.WebTestMixin()
+    try:
+        wtm._patch_settings()
+        yield django_webtest.DjangoTestApp()
+    finally:
+        wtm._unpatch_settings

tests/olap.model

@@ -133,6 +133,15 @@
                "name" : "count",
                "type" : "integer"
             },
+            {
+               "expression" : "count(evolution.id)",
+               "join" : [
+                  "evolution"
+               ],
+               "label" : "nombre de demandes ayant changée de statut",
+               "name" : "count",
+               "type" : "integer"
+            },
             {
                "expression" : "avg(endpoint_delay)",
                "label" : "délai de traitement moyen",
@@ -228,6 +237,17 @@
                "value" : "status.id",
                "value_label" : "status.label"
             },
+            {
+               "join" : [
+                  "evolution",
+                  "evolution_status"
+               ],
+               "label" : "statut dans l'historique",
+               "name" : "evolution_status",
+               "type" : "integer",
+               "value" : "evolution_status.id",
+               "value_label" : "evolution_status.label"
+            },
             {
                "filter" : false,
                "join" : [
@@ -336,6 +356,12 @@
                "name" : "status",
                "table" : "status_demande"
             },
+            {
+               "detail" : "id",
+               "master" : "evolution.status_id",
+               "name" : "evolution_status",
+               "table" : "status_demande"
+            },
             {
                "detail" : "id",
                "master" : "function__receiver",
@@ -353,6 +379,12 @@
                "master" : "\"field_itemOpen\"",
                "name" : "itemOpen",
                "table" : "formdata_demande_field_itemOpen"
+            },
+            {
+               "facts" : "formdata_id",
+               "master" : "id",
+               "name" : "evolution",
+               "table" : "evolution_demande"
             }
          ],
          "key" : "id",
@@ -364,6 +396,15 @@
                "name" : "count",
                "type" : "integer"
             },
+            {
+               "expression" : "count(evolution.id)",
+               "join" : [
+                  "evolution"
+               ],
+               "label" : "nombre de demandes ayant changée de statut",
+               "name" : "count",
+               "type" : "integer"
+            },
             {
                "expression" : "avg(endpoint_delay)",
                "label" : "délai de traitement moyen",
@@ -393,6 +434,51 @@
                "label" : "localisation géographique",
                "name" : "geolocation",
                "type" : "point"
+            },
+            {
+               "expression" : "MAX(COALESCE(evolution.delay, NOW() - {fact_table}.receipt_time2)) FILTER (WHERE evolution.status_id = 0)",
+               "join" : [
+                  "evolution"
+               ],
+               "label" : "délai maximum avant le statut Just Submitted",
+               "name" : "max_delay_until_0_Just_Submitted",
+               "type" : "duration"
+            },
+            {
+               "expression" : "MAX(COALESCE(evolution.delay, NOW() - {fact_table}.receipt_time2)) FILTER (WHERE evolution.status_id = 1)",
+               "join" : [
+                  "evolution"
+               ],
+               "label" : "délai maximum avant le statut New",
+               "name" : "max_delay_until_1_New",
+               "type" : "duration"
+            },
+            {
+               "expression" : "MAX(COALESCE(evolution.delay, NOW() - {fact_table}.receipt_time2)) FILTER (WHERE evolution.status_id = 2)",
+               "join" : [
+                  "evolution"
+               ],
+               "label" : "délai maximum avant le statut Rejected",
+               "name" : "max_delay_until_2_Rejected",
+               "type" : "duration"
+            },
+            {
+               "expression" : "MAX(COALESCE(evolution.delay, NOW() - {fact_table}.receipt_time2)) FILTER (WHERE evolution.status_id = 3)",
+               "join" : [
+                  "evolution"
+               ],
+               "label" : "délai maximum avant le statut Accepted",
+               "name" : "max_delay_until_3_Accepted",
+               "type" : "duration"
+            },
+            {
+               "expression" : "MAX(COALESCE(evolution.delay, NOW() - {fact_table}.receipt_time2)) FILTER (WHERE evolution.status_id = 4)",
+               "join" : [
+                  "evolution"
+               ],
+               "label" : "délai maximum avant le statut Finished",
+               "name" : "max_delay_until_4_Finished",
+               "type" : "duration"
             }
          ],
          "name" : "formdata_demande"

tests/test_bijoe.py

@@ -0,0 +1,8 @@
+
+from utils import login
+
+
+def test_bijoe(bijoe, admin):
+    response = login(bijoe, admin, '/')
+    import pdb
+    pdb.set_trace()

tests/test_wcs.py

@@ -1,3 +1,5 @@
+import isodate
+import pprint
 import json
 
 import pytest
@@ -25,15 +27,18 @@ def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog):
         ('evolution', 'time'),
         ('evolution', 'date'),
         ('evolution', 'hour_id'),
+        ('evolution', 'delay'),
         ('evolution_demande', 'id'),
         ('evolution_demande', 'status_id'),
         ('evolution_demande', 'formdata_id'),
         ('evolution_demande', 'time'),
         ('evolution_demande', 'date'),
         ('evolution_demande', 'hour_id'),
+        ('evolution_demande', 'delay'),
         ('formdata', 'id'),
         ('formdata', 'formdef_id'),
         ('formdata', 'receipt_time'),
+        ('formdata', 'receipt_time2'),
         ('formdata', 'hour_id'),
         ('formdata', 'channel_id'),
         ('formdata', 'backoffice'),
@@ -45,6 +50,7 @@ def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog):
         ('formdata_demande', 'id'),
         ('formdata_demande', 'formdef_id'),
         ('formdata_demande', 'receipt_time'),
+        ('formdata_demande', 'receipt_time2'),
         ('formdata_demande', 'hour_id'),
         ('formdata_demande', 'channel_id'),
         ('formdata_demande', 'backoffice'),
@@ -94,6 +100,21 @@ def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog):
             expected_json_schema['pg_dsn'] = postgres_db.dsn
             assert json_schema == expected_json_schema
 
+    with postgres_db.conn() as conn:
+        with conn.cursor() as c:
+            c.execute('SET search_path = olap')
+            c.execute('''SELECT
+        item.label,
+        COUNT(formdata.id) AS demande_count,
+        MAX(COALESCE(evolution.delay,
+                     NOW() - formdata.receipt_time2))
+            FILTER (WHERE evolution.status_id = 1) AS demande_delai
+    FROM formdata_demande AS formdata
+    LEFT OUTER JOIN formdata_demande_field_item AS item
+        ON item.id = formdata.field_item
+    LEFT OUTER JOIN evolution_demande AS evolution
+        ON evolution.formdata_id = formdata.id GROUP BY item.label''')
+
 
 def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog):
     with mock.patch('requests.get', side_effect=requests.RequestException('wat!')):

tests/utils.py

@@ -15,3 +15,74 @@ def run_wcs_script(wcs_dir, script, script_name):
     subprocess.check_call(
         [WCSCTL, 'runscript', '--app-dir', str(wcs_dir), '--vhost', HOSTNAME,
          str(script_path)])
+
+import io
+import zipfile
+import xml.etree.ElementTree as ET
+
+from django.conf import settings
+
+
+def login(app, user, path=None, password=None):
+    if path:
+        login_page = app.get(path)
+    else:
+        login_page = app.get(settings.LOGIN_URL)
+    login_page = login_page.maybe_follow()
+    form = login_page.form
+    form.set('username', user.username if hasattr(user, 'username') else user)
+    # password is supposed to be the same as username
+    form.set('password', password or user.username)
+    response = form.submit(name='login-password-submit').follow(expect_errors=True)
+    if path:
+        assert response.request.path == path
+    assert '_auth_user_id' in app.session
+    assert str(app.session['_auth_user_id']) == str(user.id)
+    return response
+
+
+def get_table(response):
+    table = []
+
+    for tr in response.pyquery('table tr'):
+        row = []
+        table.append(row)
+        for td in tr.findall('td'):
+            row.append((td.text or '').strip())
+    return table
+
+
+def xml_node_text_content(node):
+    '''Extract text content from node and all its children. Equivalent to
+       xmlNodeGetContent from libxml.'''
+
+    if node is None:
+        return ''
+
+    def helper(node):
+        s = []
+        if node.text:
+            s.append(node.text)
+        for child in node:
+            s.extend(helper(child))
+            if child.tail:
+                s.append(child.tail)
+        return s
+    return u''.join(helper(node))
+
+
+def get_ods_document(response):
+    return ET.fromstring(zipfile.ZipFile(io.BytesIO(response.content)).read('content.xml'))
+
+
+def get_ods_table(response):
+    from bijoe.visualization.ods import TABLE_NS
+
+    root = get_ods_document(response)
+    table = []
+    for row_node in root.findall('.//{%s}table-row' % TABLE_NS):
+        row = []
+        table.append(row)
+        for cell_node in row_node.findall('.//{%s}table-cell' % TABLE_NS):
+            row.append(xml_node_text_content(cell_node))
+    return table

tox.ini

@@ -5,19 +5,28 @@
 
 [tox]
 toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/wcs-olap/{env:BRANCH_NAME:}
-envlist = coverage
+envlist = py2-coverage
 
 [testenv]
 usedevelop = true
-basepython = python2
 setenv =
 	coverage: COVERAGE=--junit-xml=junit.xml --cov=wcs_olap --cov-report xml --cov-report html
 	WCSCTL=wcs/wcsctl.py
+	PGPORT={env:PGPORT:}
+	PGHOST={env:PGHOST:}
+	PGUSER={env:PGUSER:}
+	PGPASSWORD={env:PGPASSWORD:}
+	DJANGO_SETTINGS_MODULE=bijoe.settings
+	BIJOE_SETTINGS_FILE=tests/bijoe_settings.py
 deps =
 	coverage
 	pytest
 	pytest-cov
 	pytest-random
+	pytest-django
+	django-webtest
+	WebTest
+	pyquery
 	quixote<3.0
 	psycopg2-binary
 	vobject
@@ -25,6 +34,7 @@ deps =
 	django-ratelimit<3
 	gadjo
 	mock
+	git+https://git.entrouvert.org/bijoe.git@wip/29914-Pouvoir-declarer-des-jointures-n
 	django>=1.11,<1.12
 commands =
 	./get_wcs.sh

wcs_olap/feeder.py

@@ -218,6 +218,13 @@ class WcsOlapFeeder(object):
                     'type': 'integer',
                     'expression': 'count({fact_table}.id)',
                 },
+                {
+                    'name': 'count',
+                    'label': u'nombre de demandes ayant changée de statut',
+                    'type': 'integer',
+                    'expression': 'count(evolution.id)',
+                    'join': ['evolution'],
+                },
                 {
                     'name': 'avg_endpoint_delay',
                     'label': 'délai de traitement moyen',
@@ -468,6 +475,7 @@ CREATE TABLE public.dates AS (SELECT
             ['id', 'serial primary key'],
             ['formdef_id', 'smallint REFERENCES {form_table} (id)'],
             ['receipt_time', 'date'],
+            ['receipt_time2', 'timestamp'],
             ['hour_id', 'smallint REFERENCES {hour_table} (id)'],
             ['channel_id', 'smallint REFERENCES {channel_table} (id)'],
             ['backoffice', 'boolean'],
@@ -500,6 +508,7 @@ CREATE TABLE public.dates AS (SELECT
             ['time', 'timestamp'],
             ['date', 'date'],
             ['hour_id', 'smallint REFERENCES {hour_table} (id)'],
+            ['delay', 'interval'],
         ])
         self.ex('COMMENT ON TABLE {generic_evolution_table} IS %s', vars=(u'evolution générique',))
 
@@ -687,6 +696,7 @@ class WcsFormdefFeeder(object):
             ['time', 'timestamp'],
             ['date', 'date'],
             ['hour_id', 'smallint REFERENCES {hour_table} (id)'],
+            ['delay', 'interval'],
         ])
         self.ex('COMMENT ON TABLE "{evolution_table}" IS %s',
                 vars=(u'evolution des demandes %s' % self.formdef.schema.name,))
@@ -754,6 +764,8 @@ class WcsFormdefFeeder(object):
             row = {
                 'formdef_id': self.formdef_sql_id,
                 'receipt_time': data.receipt_time,
+                # anonymise a little the timestamp by truncating precision to hour
+                'receipt_time2': data.receipt_time.replace(minute=0, second=0, microsecond=0),
                 'hour_id': data.receipt_time.hour,
                 'channel_id': self.channel_to_id[channel],
                 'backoffice': data.submission.backoffice,
@@ -826,11 +838,11 @@ class WcsFormdefFeeder(object):
                 status_id = self.status_mapping[status.id]
                 generic_status_id = self.generic_status(status)
                 evolution.append(
-                    [0, status_id, evo.time, evo.time.date(), evo.time.hour])
+                    [0, status_id, evo.time, evo.time.date(), evo.time.hour, evo.delay])
                 if generic_status_id == last_status:
                     continue
                 generic_evolution.append(
-                    [0, generic_status_id, evo.time, evo.time.date(), evo.time.hour])
+                    [0, generic_status_id, evo.time, evo.time.date(), evo.time.hour, evo.delay])
                 last_status = generic_status_id
             generic_evolution_values.append(generic_evolution)
             evolution_values.append(evolution)
@@ -849,12 +861,12 @@ class WcsFormdefFeeder(object):
                 generic_evolutions.append(tuple(row))
                 if len(generic_evolutions) == 500:
                     self.ex('INSERT INTO {generic_evolution_table} (%s) VALUES %s' % (
-                        ', '.join(['formdata_id', 'generic_status_id', 'time', 'date', 'hour_id']),
+                        ', '.join(['formdata_id', 'generic_status_id', 'time', 'date', 'hour_id', 'delay']),
                         ', '.join(['%s'] * len(generic_evolutions))), vars=generic_evolutions)
                     generic_evolutions = []
         if generic_evolutions:
             self.ex('INSERT INTO {generic_evolution_table} (%s) VALUES %s' % (
-                ', '.join(['formdata_id', 'generic_status_id', 'time', 'date', 'hour_id']),
+                ', '.join(['formdata_id', 'generic_status_id', 'time', 'date', 'hour_id', 'delay']),
                 ', '.join(['%s'] * len(generic_evolutions))), vars=generic_evolutions)
 
         # insert evolutions
@@ -865,12 +877,12 @@ class WcsFormdefFeeder(object):
                 evolutions.append(tuple(row))
                 if len(evolutions) == 500:
                     self.ex('INSERT INTO "{evolution_table}" (%s) VALUES %s' % (
-                       ', '.join(['formdata_id', 'status_id', 'time', 'date', 'hour_id']),
-                       ', '.join(['%s'] * len(evolutions))), vars=evolutions)
+                        ', '.join(['formdata_id', 'status_id', 'time', 'date', 'hour_id', 'delay']),
+                        ', '.join(['%s'] * len(evolutions))), vars=evolutions)
                     evolutions = []
         if evolutions:
             self.ex('INSERT INTO "{evolution_table}" (%s) VALUES %s' % (
-                ', '.join(['formdata_id', 'status_id', 'time', 'date', 'hour_id']),
+                ', '.join(['formdata_id', 'status_id', 'time', 'date', 'hour_id', 'delay']),
                 ', '.join(['%s'] * len(evolutions))), vars=evolutions)
 
     def get_first_agent_in_evolution(self, formdata):
@@ -913,6 +925,20 @@ class WcsFormdefFeeder(object):
             'value': 'status.id',
             'value_label': 'status.label',
         })
+        cube['joins'].append({
+            'name': 'evolution_status',
+            'table': self.status_table_name,
+            'master': 'evolution.status_id',
+            'detail': 'id',
+        })
+        cube['dimensions'].append({
+            'name': 'evolution_status',
+            'label': 'statut dans l\'historique',
+            'join': ['evolution', 'evolution_status'],
+            'type': 'integer',
+            'value': 'evolution_status.id',
+            'value_label': 'evolution_status.label',
+        })
 
         # add dimension for function
         for function, name in self.formdef.schema.workflow.functions.iteritems():
@@ -992,6 +1018,27 @@ class WcsFormdefFeeder(object):
                 cube['joins'].append(join)
             cube['dimensions'].append(dimension)
 
+        # add join for evolutions
+        cube['joins'].append({
+            'name': 'evolution',
+            'table': self.evolution_table_name,
+            'master': 'id',
+            'facts': 'formdata_id',
+        })
+
+        # add measure of delay for each status since receipt_time
+        for status_id, status in enumerate(self.formdef.schema.workflow.statuses):
+            cube['measures'].append(
+                {
+                    'name': 'max_delay_until_%s_%s' % (status_id, slugify(status.name)),
+                    'label': u'délai maximum avant le statut %s' % status.name,
+                    'type': 'duration',
+                    'expression': 'MAX(COALESCE(evolution.delay, NOW() - {fact_table}.receipt_time2)) FILTER (WHERE evolution.status_id = %s)' % status_id,
+                    'join': ['evolution'],
+                }
+            )
+
+
         self.model['cubes'].append(cube)
         if self.do_feed:
             try:

wcs_olap/wcs_api.py

@@ -79,15 +79,23 @@ class Evolution(BaseObject):
     who = None
     status = None
     parts = None
+    formdata = None
 
-    def __init__(self, wcs_api, **kwargs):
+    def __init__(self, wcs_api, formdata, **kwargs):
         super(Evolution, self).__init__(wcs_api, **kwargs)
+        self.formdata = formdata
         self.time = isodate.parse_datetime(self.time)
         if self.parts:
             self.parts = [BaseObject(wcs_api, **part) for part in self.parts]
         if self.who:
             self.who = EvolutionUser(wcs_api, **self.who)
 
+    @property
+    def delay(self):
+        '''Compute delay as the time when the last not endpoint status precedes an endpoint
+           status.'''
+        return self.time - self.formdata.receipt_time
+
 
 class FormData(BaseObject):
     geolocations = None
@@ -98,7 +106,7 @@ class FormData(BaseObject):
         self.receipt_time = isodate.parse_datetime(self.receipt_time)
         self.submission = BaseObject(wcs_api, **self.submission)
         self.workflow = FormDataWorkflow(wcs_api, **self.workflow)
-        self.evolution = [Evolution(wcs_api, **evo) for evo in self.evolution or []]
+        self.evolution = [Evolution(wcs_api, self, **evo) for evo in self.evolution or []]
         self.functions = {}
         self.concerned_roles = []
         self.action_roles = []