entrouvert/wcs
entrouvert
/
wcs
14
1
Fork 0
Code Issues Pull Requests 34 Releases Activity

general: add snapshot objects (#4960)

This commit is contained in:
Frédéric Péters 2020-08-10 13:10:04 +02:00
parent dda8482b53
commit 619c5fbb57
12 changed files with 245 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/test_formdef.py
View File

@ -246,12 +246,16 @@ def test_internal_identifier_migration(pub):
def test_page_field_migration(pub):
# disable snapshots as XML serialization of old post_conditions structure
# is not possible.
pub.snapshot_class = None
FormDef.wipe()
formdef = FormDef()
formdef.name = 'foo'
formdef.fields = [
PageField(id='1', type='page',
condition='foo',
# old post_conditions structure
post_conditions=[{'condition': 'blah', 'error_message': 'blah'}]),
]
formdef.store()

61
tests/test_snapshots.py Normal file
View File

@ -0,0 +1,61 @@
import pytest
from wcs.carddef import CardDef
from wcs.formdef import FormDef
from wcs.data_sources import NamedDataSource
from utilities import get_app, login, create_temporary_pub, clean_temporary_pub
@pytest.fixture
def pub(request, emails):
pub = create_temporary_pub(sql_mode=True, templates_mode=True, lazy_mode=True,)
pub.cfg['identification'] = {'methods': ['password']}
pub.cfg['language'] = {'language': 'en'}
pub.write_cfg()
return pub
def teardown_module(module):
clean_temporary_pub()
def test_snapshot_basics(pub):
FormDef.wipe()
CardDef.wipe()
NamedDataSource.wipe()
formdef = FormDef()
formdef.name = 'testform'
formdef.fields = []
formdef.store()
carddef = CardDef()
carddef.name = 'testcard'
carddef.fields = []
carddef.store()
formdef.name = 'testform2'
formdef.store()
carddef.name = 'testcard2'
carddef.store()
data_source = NamedDataSource(name='foobar')
data_source.data_source = {'type': 'formula',
'value': repr([('1', 'un'), ('2', 'deux')])}
data_source.store()
assert pub.snapshot_class.count() == 5
# check calling .store() without changes doesn't create snapshots
# (this cannot be done on formdefs/carddefs as they embed a modification
# time)
data_source.store()
assert pub.snapshot_class.count() == 5
# check we got correct data in the serializations
snapshot = pub.snapshot_class.get_latest('formdef', formdef.id)
assert '>testform2<' in snapshot.serialization
snapshot = pub.snapshot_class.get_latest('carddef', carddef.id)
assert '>testcard2<' in snapshot.serialization

3
tests/utilities.py
View File

@ -80,12 +80,14 @@ def create_temporary_pub(sql_mode=False, templates_mode=False, lazy_mode=False):
pub.tracking_code_class = sql.TrackingCode
pub.session_class = sql.Session
pub.custom_view_class = sql.CustomView
pub.snapshot_class = sql.Snapshot
pub.is_using_postgresql = lambda: True
else:
pub.user_class = User
pub.tracking_code_class = TrackingCode
pub.session_class = sessions.BasicSession
pub.custom_view_class = custom_views.CustomView
pub.snapshot_class = None
pub.is_using_postgresql = lambda: False
pub.session_manager_class = sessions.StorageSessionManager
@ -168,6 +170,7 @@ def create_temporary_pub(sql_mode=False, templates_mode=False, lazy_mode=False):
sql.do_tracking_code_table()
sql.do_session_table()
sql.do_custom_views_table()
sql.do_snapshots_table()
sql.do_meta_table()
conn.close()

4
wcs/blocks.py
View File

@ -56,7 +56,7 @@ class BlockDef(StorableObject):
self.name = name
self.fields = []
def store(self):
def store(self, comment=None):
assert not self.is_readonly()
if self.slug is None:
# set slug if it's not yet there
@ -69,6 +69,8 @@ class BlockDef(StorableObject):
self.last_modification_user_id = None
super().store()
if get_publisher().snapshot_class:
get_publisher().snapshot_class.snap(instance=self, comment=comment)
def get_new_slug(self):
new_slug = misc.simplify(self.name, space='_')

4
wcs/carddef.py
View File

@ -131,6 +131,6 @@ class CardDef(FormDef):
base_url = get_publisher().get_frontoffice_url()
return '%s/api/cards/%s/' % (base_url, self.url_name)
def store(self):
def store(self, comment=None):
self.roles = self.backoffice_submission_roles
return super(CardDef, self).store()
return super().store(comment=comment)

4
wcs/data_sources.py
View File

@ -355,12 +355,14 @@ class NamedDataSource(XmlStorableObject):
if changed:
self.store()
def store(self):
def store(self, comment=None):
assert not self.is_readonly()
if self.slug is None:
# set slug if it's not yet there
self.slug = self.get_new_slug()
super(NamedDataSource, self).store()
if get_publisher().snapshot_class:
get_publisher().snapshot_class.snap(instance=self, comment=comment)
def get_new_slug(self):
new_slug = misc.simplify(self.name, space='_')

4
wcs/formdef.py
View File

@ -359,7 +359,7 @@ class FormDef(StorableObject):
from . import sql
sql.formdef_wipe()
def store(self):
def store(self, comment=None):
assert not self.is_readonly()
if self.url_name is None:
# set url name if it's not yet there
@ -379,6 +379,8 @@ class FormDef(StorableObject):
else:
self.last_modification_user_id = None
t = StorableObject.store(self)
if get_publisher().snapshot_class:
get_publisher().snapshot_class.snap(instance=self, comment=comment)
if get_publisher().is_using_postgresql():
from . import sql
sql.do_formdef_tables(self, rebuild_views=True,

3
wcs/publisher.py
View File

@ -150,12 +150,14 @@ class WcsPublisher(StubWcsPublisher):
self.tracking_code_class = sql.TrackingCode
self.session_class = sql.Session
self.custom_view_class = sql.CustomView
self.snapshot_class = sql.Snapshot
sql.get_connection(new=True)
else:
self.user_class = User
self.tracking_code_class = TrackingCode
self.session_class = sessions.BasicSession
self.custom_view_class = custom_views.CustomView
self.snapshot_class = None
self.session_manager_class = sessions.StorageSessionManager
self.set_session_manager(self.session_manager_class(session_class=self.session_class))
@ -306,6 +308,7 @@ class WcsPublisher(StubWcsPublisher):
sql.do_user_table()
sql.do_tracking_code_table()
sql.do_custom_views_table()
sql.do_snapshots_table()
sql.do_meta_table()
from .formdef import FormDef
from .carddef import CardDef

47
wcs/snapshots.py Normal file
View File

@ -0,0 +1,47 @@
# w.c.s. - web application for online forms
# Copyright (C) 2005-2020 Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
import xml.etree.ElementTree as ET
from django.utils.timezone import now
from quixote import get_session
class Snapshot:
id = None
object_type = None # (formdef, carddef, blockdef, workflow, data_source, etc.)
object_id = None
timestamp = None
user_id = None
comment = None
serialization = None
label = None # (named snapshot)
@classmethod
def snap(cls, instance, comment):
obj = cls()
obj.object_type = instance.xml_root_node
obj.object_id = instance.id
obj.timestamp = now()
if get_session():
obj.user_id = get_session().user
obj.serialization = ET.tostring(instance.export_to_xml(include_id=True)).decode('utf-8')
obj.comment = comment
latest = cls.get_latest(obj.object_type, obj.object_id)
if latest is None or obj.serialization != latest.serialization:
# save snapshot if there are changes
obj.store()

110
wcs/sql.py
View File

@ -45,6 +45,7 @@ import wcs.categories
import wcs.carddata
import wcs.custom_views
import wcs.formdata
import wcs.snapshots
import wcs.tracking_code
import wcs.users
@ -818,6 +819,38 @@ def do_custom_views_table():
cur.close()
def do_snapshots_table():
conn, cur = get_connection_and_cursor()
table_name = 'snapshots'
cur.execute('''SELECT COUNT(*) FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = %s''', (table_name,))
if cur.fetchone()[0] == 0:
cur.execute('''CREATE TABLE %s (id SERIAL,
object_type VARCHAR,
object_id VARCHAR,
timestamp TIMESTAMP WITH TIME ZONE,
user_id VARCHAR,
comment TEXT,
serialization TEXT,
label VARCHAR
)''' % table_name)
cur.execute('''SELECT column_name FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = %s''', (table_name,))
existing_fields = set([x[0] for x in cur.fetchall()])
needed_fields = set([x[0] for x in Snapshot._table_static_fields])
# delete obsolete fields
for field in (existing_fields - needed_fields):
cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
conn.commit()
cur.close()
@guard_postgres
def do_meta_table(conn=None, cur=None, insert_current_sql_level=True):
own_conn = False
@ -2330,6 +2363,78 @@ class CustomView(SqlMixin, wcs.custom_views.CustomView):
return []
class Snapshot(SqlMixin, wcs.snapshots.Snapshot):
_table_name = 'snapshots'
_table_static_fields = [
('id', 'serial'),
('object_type', 'varchar'),
('object_id', 'varchar'),
('timestamp', 'timestamptz'),
('user_id', 'varchar'),
('comment', 'text'),
('serialization', 'text'),
('label', 'varchar'),
]
@guard_postgres
@invalidate_substitution_cache
def store(self):
sql_dict = {x: getattr(self, x) for x, y in self._table_static_fields}
conn, cur = get_connection_and_cursor()
if not self.id:
column_names = [x for x in sql_dict.keys() if x != 'id']
sql_statement = '''INSERT INTO %s (%s)
VALUES (%s)
RETURNING id''' % (
self._table_name,
', '.join(column_names),
', '.join(['%%(%s)s' % x for x in column_names]))
cur.execute(sql_statement, sql_dict)
self.id = cur.fetchone()[0]
else:
column_names = sql_dict.keys()
sql_statement = '''UPDATE %s SET %s WHERE id = %%(id)s RETURNING id''' % (
self._table_name,
', '.join(['%s = %%(%s)s' % (x, x) for x in column_names]))
cur.execute(sql_statement, sql_dict)
if cur.fetchone() is None:
raise AssertionError()
conn.commit()
cur.close()
@classmethod
def _row2ob(cls, row, **kwargs):
o = cls()
for field, value in zip(cls._table_static_fields, tuple(row)):
if field[1] in ('serial', 'timestamptz'):
setattr(o, field[0], value)
elif field[1] in ('varchar', 'text'):
setattr(o, field[0], str_encode(value))
return o
@classmethod
def get_data_fields(cls):
return []
@classmethod
def get_latest(cls, object_type, object_id):
conn, cur = get_connection_and_cursor()
sql_statement = '''SELECT id FROM snapshots
WHERE object_type = %(object_type)s
AND object_id = %(object_id)s
ORDER BY timestamp DESC
LIMIT 1'''
cur.execute(sql_statement, {'object_type': object_type, 'object_id': object_id})
row = cur.fetchone()
conn.commit()
cur.close()
if row is None:
return None
return cls.get(row[0])
class classproperty(object):
def __init__(self, f):
self.f = f
@ -2560,7 +2665,7 @@ def get_yearly_totals(period_start=None, period_end=None, criterias=None):
return result
SQL_LEVEL = 41
SQL_LEVEL = 42
def migrate_global_views(conn, cur):
@ -2708,6 +2813,9 @@ def migrate():
for evo in formdata.evolution:
evo.who = None
formdata.store()
if sql_level < 42:
# 42: create snapshots table
do_snapshots_table()
cur.execute('''UPDATE wcs_meta SET value = %s WHERE key = %s''', (
str(SQL_LEVEL), 'sql_level'))

5
wcs/workflows.py
View File

@ -308,6 +308,7 @@ class WorkflowBackofficeFieldsFormDef(FormDef):
class Workflow(StorableObject):
_names = 'workflows'
xml_root_node = 'workflow'
name = None
possible_status = None
roles = None
@ -347,7 +348,7 @@ class Workflow(StorableObject):
if changed:
self.store()
def store(self):
def store(self, comment=None):
assert not self.is_readonly()
must_update = False
if self.id:
@ -378,6 +379,8 @@ class Workflow(StorableObject):
else:
self.last_modification_user_id = None
StorableObject.store(self)
if get_publisher().snapshot_class:
get_publisher().snapshot_class.snap(instance=self, comment=comment)
def update(job=None):
# instruct all related formdefs to update.

4
wcs/wscalls.py
View File

@ -218,7 +218,7 @@ class NamedWsCall(XmlStorableObject):
request['post_formdata'] = bool(element.find('post_formdata') is not None)
return request
def store(self):
def store(self, comment=None):
assert not self.is_readonly()
if self.slug is None:
# set slug if it's not yet there
@ -227,6 +227,8 @@ class NamedWsCall(XmlStorableObject):
self.remove_object(self.id)
self.id = self.slug
super(NamedWsCall, self).store()
if get_publisher().snapshot_class:
get_publisher().snapshot_class.snap(instance=self, comment=comment)
def get_new_slug(self):
new_slug = simplify(self.name, space='_')