entrouvert
/
wcs-olap
16
0
Fork 0
Code Issues Pull Requests Releases Activity

add formdata.json_data column (fixes #14646) 

It will be filled with unstructred representation of forms'datas.
This commit is contained in:
Benjamin Dauvergne 2017-01-18 09:58:00 +01:00
parent a28b57fe53
commit 0bf1faed1f
1 changed files with 32 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
wcs_olap/feeder.py
View File

@ -85,6 +85,7 @@ class WcsOlapFeeder(object):
'name': 'all_formdata',
'label': u'Tous les formulaires',
'fact_table': 'formdata',
'json_field': 'json_data',
'key': 'id',
'joins': [
{
@ -232,6 +233,7 @@ class WcsOlapFeeder(object):
self.model['cubes'].append(cube)
self.base_cube = self.model['cubes'][0]
self.agents_mapping = {}
self.formdata_json_index = []
def hash_table_name(self, table_name):
table_name = table_name.format(**self.default_ctx)
@ -365,6 +367,7 @@ class WcsOlapFeeder(object):
['generic_status_id', 'smallint REFERENCES {generic_status_table} (id)'],
['endpoint_delay', 'interval'],
['first_agent_id', 'smallint REFERENCES {agent_table} (id)'],
['json_data', 'JSONB NULL'],
]
self.comments = {
'formdef_id': u'formulaire',
@ -431,6 +434,15 @@ class WcsOlapFeeder(object):
self.agents_mapping[user.id] = self.insert_agent(user.name)
return self.agents_mapping[user.id]
def create_formdata_json_index(self, varname):
if varname in self.formdata_json_index:
return
index_name = self.hash_table_name('{formdata_table}_%s_json_idx' % varname)
self.ex('CREATE INDEX {index_name} ON {generic_formdata_table} (("json_data"->>%s))',
ctx={'index_name': index_name}, vars=[varname])
# prevent double creation
self.formdata_json_index.append(varname)
class WcsFormdefFeeder(object):
def __init__(self, olap_feeder, formdef, do_feed=True):
@ -536,6 +548,12 @@ class WcsFormdefFeeder(object):
comment=u'formulaire %s' % self.formdef.schema.name)
for at, comment in comments.iteritems():
self.ex('COMMENT ON COLUMN {formdata_table}.%s IS %%s' % at, vars=(comment,))
# Creat index for JSON fields
for field in fields:
if field.varname:
self.create_formdata_json_index(field.varname)
# PostgreSQL does not propagate foreign key constraints to child tables
# so we must recreate them manually
for name, _type in self.olap_feeder.columns:
@ -586,6 +604,7 @@ class WcsFormdefFeeder(object):
generic_evolution_values = []
evolution_values = []
for data in self.formdef.datas:
json_data = {}
# ignore formdata without status
if data.workflow.status:
@ -618,20 +637,25 @@ class WcsFormdefFeeder(object):
# add form fields value
for field in self.fields:
v = None
raw = None
if field.type == 'item':
if field.varname in data.fields:
v = data.fields[field.varname]
raw = data.fields[field.varname]
elif field.varname in data.workflow.fields:
v = data.workflow.fields[field.varname]
raw = data.workflow.fields[field.varname]
else:
v = None
raw = None
# map items to sql id
if field.items or field.options:
# map items to sql id
v = self.items_mappings[field.varname].get(v)
v = self.items_mappings[field.varname].get(raw)
else:
v = v and self.get_item_id(field, v)
v = raw and self.get_item_id(field, raw)
# unstructured storage of field values
if field.varname and raw:
json_data[field.varname] = raw
row['field_%s' % field.varname] = v
row['json_data'] = json.dumps(json_data)
# add geolocation fields value
for geolocation, label in self.formdef.schema.geolocations:
v = (data.geolocations or {}).get(geolocation)
@ -679,9 +703,8 @@ class WcsFormdefFeeder(object):
if not values:
self.logger.warning('no data')
return
self.ex('INSERT INTO {formdata_table} (%s) VALUES %s RETURNING id' % (
', '.join(self.columns[1:]), # skip the id column
', '.join(values)))
self.ex('INSERT INTO {formdata_table} ({columns}) VALUES {values} RETURNING id',
ctx=dict(columns=', '.join(self.columns[1:]), values=', '.join(values)))
# insert generic evolutions
generic_evolutions = []
ids = list(self.cur.fetchall())