entrouvert
/
wcs-olap
16
0
Fork 0
Code Issues Pull Requests Releases Activity

feeder: feed evolutions to postgresql in reasonable chunks (#23399)

This commit is contained in:
Frédéric Péters 2018-04-25 11:06:15 +02:00 committed by Thomas NOEL
parent b43eaa2eb5
commit bfe7b73cb4
1 changed files with 20 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
wcs_olap/feeder.py
View File

@ -748,6 +748,7 @@ class WcsFormdefFeeder(object):
return
self.ex('INSERT INTO {formdata_table} ({columns}) VALUES {values} RETURNING id',
ctx=dict(columns=', '.join(self.columns[1:]), values=', '.join(values)))
# insert generic evolutions
generic_evolutions = []
ids = list(self.cur.fetchall())
@ -755,18 +756,31 @@ class WcsFormdefFeeder(object):
for row in evos:
row[0] = formdata_id
generic_evolutions.append(tuple(row))
self.ex('INSERT INTO {generic_evolution_table} (%s) VALUES %s' % (
', '.join(['formdata_id', 'generic_status_id', 'time', 'date', 'hour_id']),
', '.join(['%s'] * len(generic_evolutions))), vars=generic_evolutions)
if len(generic_evolutions) == 500:
self.ex('INSERT INTO {generic_evolution_table} (%s) VALUES %s' % (
', '.join(['formdata_id', 'generic_status_id', 'time', 'date', 'hour_id']),
', '.join(['%s'] * len(generic_evolutions))), vars=generic_evolutions)
generic_evolutions = []
if generic_evolutions:
self.ex('INSERT INTO {generic_evolution_table} (%s) VALUES %s' % (
', '.join(['formdata_id', 'generic_status_id', 'time', 'date', 'hour_id']),
', '.join(['%s'] * len(generic_evolutions))), vars=generic_evolutions)
# insert evolutions
evolutions = []
for evos, (formdata_id,) in zip(evolution_values, ids):
for row in evos:
row[0] = formdata_id
evolutions.append(tuple(row))
self.ex('INSERT INTO {evolution_table} (%s) VALUES %s' % (
', '.join(['formdata_id', 'status_id', 'time', 'date', 'hour_id']),
', '.join(['%s'] * len(evolutions))), vars=evolutions)
if len(evolutions) == 500:
self.ex('INSERT INTO {evolution_table} (%s) VALUES %s' % (
', '.join(['formdata_id', 'status_id', 'time', 'date', 'hour_id']),
', '.join(['%s'] * len(evolutions))), vars=evolutions)
evolutions = []
if evolutions:
self.ex('INSERT INTO {evolution_table} (%s) VALUES %s' % (
', '.join(['formdata_id', 'status_id', 'time', 'date', 'hour_id']),
', '.join(['%s'] * len(evolutions))), vars=evolutions)
def get_first_agent_in_evolution(self, formdata):
for evo in formdata.evolution: