2016-01-09 22:04:36 +01:00
|
|
|
import requests
|
|
|
|
import urlparse
|
|
|
|
import urllib
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
import isodate
|
2016-01-09 22:04:36 +01:00
|
|
|
|
|
|
|
from . import signature
|
|
|
|
|
|
|
|
|
|
|
|
class WcsApiError(Exception):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
class BaseObject(object):
|
|
|
|
def __init__(self, wcs_api, **kwargs):
|
|
|
|
self.__wcs_api = wcs_api
|
|
|
|
self.__dict__.update(**kwargs)
|
|
|
|
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
|
|
|
|
class FormDataWorkflow(BaseObject):
|
|
|
|
status = None
|
|
|
|
|
|
|
|
def __init__(self, wcs_api, **kwargs):
|
|
|
|
super(FormDataWorkflow, self).__init__(wcs_api, **kwargs)
|
|
|
|
if self.status is not None:
|
|
|
|
self.status = BaseObject(wcs_api, **self.status)
|
|
|
|
|
|
|
|
|
|
|
|
class Evolution(BaseObject):
|
|
|
|
user = None
|
|
|
|
status = None
|
|
|
|
parts = None
|
|
|
|
|
|
|
|
def __init__(self, wcs_api, **kwargs):
|
|
|
|
super(Evolution, self).__init__(wcs_api, **kwargs)
|
|
|
|
self.time = isodate.parse_datetime(self.time)
|
|
|
|
if self.parts:
|
|
|
|
self.parts = [BaseObject(wcs_api, **part) for part in self.parts]
|
2016-01-09 22:04:36 +01:00
|
|
|
|
|
|
|
|
|
|
|
class FormData(BaseObject):
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
geolocations = None
|
|
|
|
evolution = None
|
|
|
|
|
|
|
|
def __init__(self, wcs_api, **kwargs):
|
|
|
|
super(FormData, self).__init__(wcs_api, **kwargs)
|
|
|
|
self.receipt_time = isodate.parse_datetime(self.receipt_time)
|
|
|
|
self.submission = BaseObject(wcs_api, **self.submission)
|
|
|
|
self.workflow = FormDataWorkflow(wcs_api, **self.workflow)
|
|
|
|
self.evolution = [Evolution(wcs_api, **evo) for evo in self.evolution or []]
|
|
|
|
self.functions = {}
|
|
|
|
self.concerned_roles = []
|
|
|
|
self.action_roles = []
|
|
|
|
for function in self.roles:
|
|
|
|
roles = [Role(wcs_api, **r) for r in self.roles[function]]
|
|
|
|
if function == 'concerned':
|
|
|
|
self.concerned_roles.extend(roles)
|
|
|
|
elif function == 'actions':
|
|
|
|
self.concerned_roles.extend(roles)
|
|
|
|
else:
|
|
|
|
try:
|
|
|
|
self.functions[function] = roles[0]
|
|
|
|
except IndexError:
|
|
|
|
self.functions[function] = None
|
|
|
|
del self.roles
|
2016-01-09 22:04:36 +01:00
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return '<{klass} {display_id!r}>'.format(klass=self.__class__.__name__,
|
|
|
|
display_id=self.id)
|
|
|
|
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
@property
|
|
|
|
def endpoint_delay(self):
|
|
|
|
'''Compute delay as the time when the last not endpoint status precedes an endpoint
|
|
|
|
status.'''
|
|
|
|
statuses_map = self.formdef.schema.workflow.statuses_map
|
|
|
|
s = 0
|
|
|
|
for evo in self.evolution[::-1]:
|
|
|
|
if evo.status:
|
2016-05-19 14:40:21 +02:00
|
|
|
try:
|
|
|
|
status = statuses_map[evo.status]
|
|
|
|
except KeyError: # happen when workflow has changed
|
|
|
|
return
|
|
|
|
if status.endpoint:
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
s = 1
|
|
|
|
last = evo.time - self.receipt_time
|
|
|
|
else:
|
|
|
|
if s == 1:
|
|
|
|
return last
|
|
|
|
else:
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
class Workflow(BaseObject):
|
|
|
|
status = None
|
|
|
|
|
|
|
|
def __init__(self, wcs_api, **kwargs):
|
|
|
|
super(Workflow, self).__init__(wcs_api, **kwargs)
|
|
|
|
self.statuses = [BaseObject(wcs_api, **v) for v in self.statuses]
|
|
|
|
self.statuses_map = dict((s.id, s) for s in self.statuses)
|
|
|
|
|
|
|
|
|
|
|
|
class Field(BaseObject):
|
|
|
|
items = None
|
|
|
|
options = None
|
|
|
|
varname = None
|
|
|
|
in_filters = False
|
|
|
|
|
|
|
|
|
|
|
|
class Schema(BaseObject):
|
|
|
|
category_id = None
|
|
|
|
category = None
|
|
|
|
geolocations = None
|
|
|
|
|
|
|
|
def __init__(self, wcs_api, **kwargs):
|
|
|
|
super(Schema, self).__init__(wcs_api, **kwargs)
|
|
|
|
self.workflow = Workflow(wcs_api, **self.workflow)
|
|
|
|
self.fields = [Field(wcs_api, **f) for f in self.fields]
|
|
|
|
self.geolocations = sorted((k, v) for k, v in (self.geolocations or {}).items())
|
|
|
|
|
2016-01-09 22:04:36 +01:00
|
|
|
|
|
|
|
class FormDef(BaseObject):
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
geolocations = None
|
|
|
|
|
2016-01-09 22:04:36 +01:00
|
|
|
def __init__(self, wcs_api, **kwargs):
|
|
|
|
self.__wcs_api = wcs_api
|
|
|
|
self.__dict__.update(**kwargs)
|
|
|
|
|
|
|
|
def __unicode__(self):
|
|
|
|
return self.title
|
|
|
|
|
|
|
|
@property
|
|
|
|
def datas(self):
|
|
|
|
datas = self.__wcs_api.get_formdata(self.slug)
|
|
|
|
for data in datas:
|
|
|
|
data.formdef = self
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
yield data
|
2016-01-09 22:04:36 +01:00
|
|
|
|
|
|
|
@property
|
|
|
|
def schema(self):
|
|
|
|
return self.__wcs_api.get_schema(self.slug)
|
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return '<{klass} {slug!r}>'.format(klass=self.__class__.__name__, slug=self.slug)
|
|
|
|
|
|
|
|
|
2016-04-28 14:51:55 +02:00
|
|
|
class Role(BaseObject):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
class Category(BaseObject):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2016-01-09 22:04:36 +01:00
|
|
|
class WcsApi(object):
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
def __init__(self, url, orig, key, verify=True, slugs=None):
|
2016-01-09 22:04:36 +01:00
|
|
|
self.url = url
|
|
|
|
self.orig = orig
|
|
|
|
self.key = key
|
|
|
|
self.verify = verify
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
self.cache = {}
|
|
|
|
self.slugs = slugs or []
|
2016-01-09 22:04:36 +01:00
|
|
|
|
|
|
|
@property
|
|
|
|
def formdefs_url(self):
|
|
|
|
return urlparse.urljoin(self.url, 'api/formdefs/')
|
|
|
|
|
|
|
|
@property
|
|
|
|
def forms_url(self):
|
|
|
|
return urlparse.urljoin(self.url, 'api/forms/')
|
|
|
|
|
2016-04-28 14:51:55 +02:00
|
|
|
@property
|
|
|
|
def roles_url(self):
|
|
|
|
return urlparse.urljoin(self.url, 'api/roles')
|
|
|
|
|
2016-01-09 22:04:36 +01:00
|
|
|
def get_json(self, *url_parts):
|
|
|
|
url = reduce(lambda x, y: urlparse.urljoin(x, y), url_parts)
|
|
|
|
params = {'orig': self.orig}
|
|
|
|
query_string = urllib.urlencode(params)
|
|
|
|
presigned_url = url + ('&' if '?' in url else '?') + query_string
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
if presigned_url in self.cache:
|
|
|
|
return self.cache[presigned_url]
|
2016-01-09 22:04:36 +01:00
|
|
|
signed_url = signature.sign_url(presigned_url, self.key)
|
|
|
|
try:
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
response = requests.get(signed_url, verify=self.verify)
|
2016-01-09 22:04:36 +01:00
|
|
|
response.raise_for_status()
|
|
|
|
except requests.RequestException, e:
|
|
|
|
raise WcsApiError('GET request failed', signed_url, e)
|
|
|
|
else:
|
|
|
|
try:
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
content = response.json()
|
|
|
|
self.cache[presigned_url] = content
|
|
|
|
return content
|
2016-01-09 22:04:36 +01:00
|
|
|
except ValueError, e:
|
|
|
|
raise WcsApiError('Invalid JSON content', signed_url, e)
|
|
|
|
|
2016-04-28 14:51:55 +02:00
|
|
|
@property
|
|
|
|
def roles(self):
|
|
|
|
return [Role(wcs_api=self, **d) for d in self.get_json(self.roles_url)['data']]
|
2016-01-09 22:04:36 +01:00
|
|
|
|
2016-04-28 14:51:55 +02:00
|
|
|
@property
|
|
|
|
def formdefs(self):
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
return [FormDef(wcs_api=self, **d) for d in self.get_json(self.formdefs_url)
|
|
|
|
if not self.slugs or d['slug'] in self.slugs]
|
|
|
|
|
|
|
|
@property
|
|
|
|
def categories(self):
|
|
|
|
d = {}
|
|
|
|
for f in self.formdefs:
|
|
|
|
if hasattr(f.schema, 'category'):
|
|
|
|
d[f.schema.category_id] = f.schema.category
|
|
|
|
return [Category(wcs_api=self, id=k, name=v) for k, v in d.items()]
|
|
|
|
|
2016-01-09 22:04:36 +01:00
|
|
|
def get_formdata(self, slug):
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
for d in self.get_json(self.forms_url, slug + '/list?anonymise&full=on'):
|
2016-05-21 21:51:34 +02:00
|
|
|
# w.cs. had a bug where some formdata lost their draft status, skip them
|
|
|
|
if not d.get('receipt_time'):
|
|
|
|
continue
|
2016-04-28 14:51:55 +02:00
|
|
|
yield FormData(wcs_api=self, **d)
|
2016-01-09 22:04:36 +01:00
|
|
|
|
|
|
|
def get_schema(self, slug):
|
wcs-olap: build a start schema from data exported by w.c.s. API
Given such an .INI file:
[https://demarches.triffouilly.fr/]
orig = bi.triffouilly.fr
key = 452b8964
pg_dsn = dbname=publik-bi
email = bi@entrouvert.com
schema = triffouilly
# slugs = recette-technique-ajout-d-un-enfant
It builds a schema named 'triffouilly' in the pre-existing database named 'publik-bi', the schema will contains tables named:
channel (label varchar)
role (label varchar)
category (label varchar)
form (category, label)
formdata : parent table of all formdata tables)
(form, receipt_time, year, month, dow, hour, channel, backoffice, generic_status)
status (generic statuses: new, in progress & closed
label
for each formdef tables named:
formdata_{formdef.slug}
status_{formdef.slug}
for each anonymisable
2016-05-10 16:53:17 +02:00
|
|
|
json_schema = self.get_json(self.formdefs_url, slug + '/', 'schema?anonymise')
|
|
|
|
return Schema(wcs_api=self, **json_schema)
|