diff --git a/tests/api/test_formdata.py b/tests/api/test_formdata.py index dad4ea7f2..f93bab0c9 100644 --- a/tests/api/test_formdata.py +++ b/tests/api/test_formdata.py @@ -788,30 +788,27 @@ def test_api_list_formdata(pub, local_user): get_app(pub).get(sign_uri('/api/forms/test/list?filter=all&offset=plop', user=local_user), status=400) get_app(pub).get(sign_uri('/api/forms/test/list?filter=all&limit=plop', user=local_user), status=400) - if pub.is_using_postgresql(): - # just check ordering - resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&order_by=f0', user=local_user)) - assert [d['fields']['foobar'] for d in resp.json] == ['FOO BAR %02d' % i for i in range(0, 30)] + # just check ordering + resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&order_by=f0', user=local_user)) + assert [d['fields']['foobar'] for d in resp.json] == ['FOO BAR %02d' % i for i in range(0, 30)] - resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&order_by=-f0', user=local_user)) - assert [d['fields']['foobar'] for d in resp.json] == ['FOO BAR %02d' % i for i in range(29, -1, -1)] + resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&order_by=-f0', user=local_user)) + assert [d['fields']['foobar'] for d in resp.json] == ['FOO BAR %02d' % i for i in range(29, -1, -1)] - resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&order_by=foobar', user=local_user)) - assert [d['fields']['foobar'] for d in resp.json] == ['FOO BAR %02d' % i for i in range(0, 30)] + resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&order_by=foobar', user=local_user)) + assert [d['fields']['foobar'] for d in resp.json] == ['FOO BAR %02d' % i for i in range(0, 30)] - resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&order_by=-foobar', user=local_user)) - assert [d['fields']['foobar'] for d in resp.json] == ['FOO BAR %02d' % i for i in range(29, -1, -1)] + resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&order_by=-foobar', user=local_user)) + assert [d['fields']['foobar'] for d in resp.json] == ['FOO BAR %02d' % i for i in range(29, -1, -1)] - # check fts - resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&q=foo', user=local_user)) - assert len(resp.json) == 30 - resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&q=baz', user=local_user)) - assert len(resp.json) == 14 + # check fts + resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&q=foo', user=local_user)) + assert len(resp.json) == 30 + resp = get_app(pub).get(sign_uri('/api/forms/test/list?full=on&q=baz', user=local_user)) + assert len(resp.json) == 14 def test_api_list_formdata_order_by_rank(pub, local_user): - if not pub.is_using_postgresql(): - pytest.skip('this requires SQL') pub.role_class.wipe() role = pub.role_class(name='test') role.store() @@ -938,8 +935,8 @@ def test_api_list_formdata_string_filter(pub, local_user): params = [ ('eq', 'FOO 2', 1), ('ne', 'FOO 2', 3), - ('lt', 'FOO 2', 2 if pub.is_using_postgresql() else 3), - ('lte', 'FOO 2', 3 if pub.is_using_postgresql() else 4), + ('lt', 'FOO 2', 2), + ('lte', 'FOO 2', 3), ('gt', 'FOO 2', 0), ('gt', '42', 0), ('gte', 'FOO 2', 1), @@ -957,8 +954,8 @@ def test_api_list_formdata_string_filter(pub, local_user): ('eq', '10', 1), ('eq', '010', 1), ('ne', '10', 3), - ('lt', '10', 1 if pub.is_using_postgresql() else 2), - ('lte', '10', 2 if pub.is_using_postgresql() else 3), + ('lt', '10', 1), + ('lte', '10', 2), ('gt', '10', 1), ('gt', '9', 2), ('gte', '10', 2), @@ -1021,8 +1018,8 @@ def test_api_list_formdata_item_filter(pub, local_user): ('eq', '10', 1), ('eq', '010', 1), ('ne', '10', 3), - ('lt', '10', 1 if pub.is_using_postgresql() else 2), - ('lte', '10', 2 if pub.is_using_postgresql() else 3), + ('lt', '10', 1), + ('lte', '10', 2), ('gt', '10', 1), ('gt', '9', 2), ('gte', '10', 2), @@ -1041,8 +1038,8 @@ def test_api_list_formdata_item_filter(pub, local_user): params = [ ('eq', 'foo', 1), ('ne', 'foo', 3), - ('lt', 'foo', 2 if pub.is_using_postgresql() else 3), - ('lte', 'foo', 3 if pub.is_using_postgresql() else 4), + ('lt', 'foo', 2), + ('lte', 'foo', 3), ('gt', 'foo', 0), ('gt', '42', 0), ('gte', 'foo', 1), @@ -1182,7 +1179,7 @@ def test_api_list_formdata_bool_filter(pub, local_user): formdata.store() resp = get_app(pub).get(sign_uri('/api/forms/test/list?filter-bool=false', user=local_user)) - assert len(resp.json) == 2 if pub.is_using_postgresql() else 3 + assert len(resp.json) == 2 resp = get_app(pub).get(sign_uri('/api/forms/test/list?filter-bool=true', user=local_user)) assert len(resp.json) == 1 params = [ @@ -1209,9 +1206,6 @@ def test_api_list_formdata_bool_filter(pub, local_user): def test_api_list_formdata_date_filter(pub, local_user): - if not pub.is_using_postgresql(): - pytest.skip('this requires SQL') - pub.role_class.wipe() role = pub.role_class(name='test') role.store() @@ -1412,9 +1406,6 @@ def test_api_list_formdata_number_filter(pub, local_user): def test_api_list_formdata_block_field_filter(pub, local_user): - if not pub.is_using_postgresql(): - pytest.skip('this requires SQL') - NamedDataSource.wipe() data_source = NamedDataSource(name='foobar') data_source.data_source = { @@ -2142,11 +2133,6 @@ def test_api_global_geojson(pub, local_user): formdata.jump_status('finished') formdata.store() - if not pub.is_using_postgresql(): - resp = get_app(pub).get(sign_uri('/api/forms/geojson', user=local_user), status=404) - pytest.skip('this requires SQL') - return - # check empty content if user doesn't have the appropriate role resp = get_app(pub).get(sign_uri('/api/forms/geojson', user=local_user)) assert 'features' in resp.json @@ -2170,11 +2156,6 @@ def test_api_global_geojson(pub, local_user): @pytest.mark.parametrize('user', ['query-email', 'api-access']) @pytest.mark.parametrize('auth', ['signature', 'http-basic']) def test_api_global_listing(pub, local_user, user, auth): - if not pub.is_using_postgresql(): - resp = get_app(pub).get(sign_uri('/api/forms/geojson', user=local_user), status=404) - pytest.skip('this requires SQL') - return - pub.role_class.wipe() role = pub.role_class(name='test') role.store() @@ -2279,10 +2260,6 @@ def test_api_global_listing(pub, local_user, user, auth): def test_api_global_listing_categories_filter(pub, local_user): - if not pub.is_using_postgresql(): - pytest.skip('this requires SQL') - return - Category.wipe() category1 = Category() category1.name = 'Category 1' @@ -2391,11 +2368,6 @@ def test_api_global_listing_ignored_roles(pub, local_user): def test_api_include_anonymised(pub, local_user): - if not pub.is_using_postgresql(): - resp = get_app(pub).get(sign_uri('/api/forms/geojson', user=local_user), status=404) - pytest.skip('this requires SQL') - return - pub.role_class.wipe() role = pub.role_class(name='test') role.store() @@ -2438,10 +2410,6 @@ def test_api_include_anonymised(pub, local_user): def test_global_forms_api_user_uuid_filter(pub, local_user): - if not pub.is_using_postgresql(): - pytest.skip('this requires SQL') - return - pub.role_class.wipe() role = pub.role_class(name='test') role.store() diff --git a/tests/api/test_formdef.py b/tests/api/test_formdef.py index ee26b5300..95318921b 100644 --- a/tests/api/test_formdef.py +++ b/tests/api/test_formdef.py @@ -177,11 +177,8 @@ def test_formdef_list(pub): formdata.store() resp = get_app(pub).get(sign_uri('/api/formdefs/?include-count=on')) - if not pub.is_using_postgresql(): - assert resp.json['data'][0]['count'] == 8 - else: - # 3*4 + 2*2 + 1*1 - assert resp.json['data'][0]['count'] == 17 + # 3*4 + 2*2 + 1*1 + assert resp.json['data'][0]['count'] == 17 def test_formdef_list_categories_filter(pub): diff --git a/tests/api/test_user.py b/tests/api/test_user.py index f9d84863c..9d436d734 100644 --- a/tests/api/test_user.py +++ b/tests/api/test_user.py @@ -410,10 +410,6 @@ def test_user_forms(pub, local_user, access): def test_user_forms_limit_offset(pub, local_user): - if not pub.is_using_postgresql(): - pytest.skip('this requires SQL') - return - FormDef.wipe() formdef = FormDef() formdef.name = 'test limit offset' @@ -614,10 +610,6 @@ def test_user_forms_api_access_restrict_to_anonymised_data(pub, local_user, acce def test_user_forms_include_accessible(pub, local_user, access): - if not pub.is_using_postgresql(): - pytest.skip('this requires SQL') - return - pub.role_class.wipe() role = pub.role_class(name='Foo bar') role.store() diff --git a/tests/backoffice_pages/test_all.py b/tests/backoffice_pages/test_all.py index e9fb9e225..96ccce2ca 100644 --- a/tests/backoffice_pages/test_all.py +++ b/tests/backoffice_pages/test_all.py @@ -269,25 +269,22 @@ def test_backoffice_forms(pub): assert '9 open on 50' in resp.text # anonymise some formdata, they should no longer be included - if pub.is_using_postgresql(): - formdef = FormDef.get_by_urlname('form-title') - for i, formdata in enumerate( - formdef.data_class().select([st.Equal('status', 'wf-finished')], order_by='id') - ): - if i >= 20: - break - formdata.anonymise() + formdef = FormDef.get_by_urlname('form-title') + for i, formdata in enumerate( + formdef.data_class().select([st.Equal('status', 'wf-finished')], order_by='id') + ): + if i >= 20: + break + formdata.anonymise() - for i, formdata in enumerate( - formdef.data_class().select([st.Equal('status', 'wf-new')], order_by='id') - ): - if i >= 5: - break - formdata.anonymise() + for i, formdata in enumerate(formdef.data_class().select([st.Equal('status', 'wf-new')], order_by='id')): + if i >= 5: + break + formdata.anonymise() - resp = app.get('/backoffice/management/forms') - assert 'Forms in your care' in resp.text - assert '4 open on 25' in resp.text + resp = app.get('/backoffice/management/forms') + assert 'Forms in your care' in resp.text + assert '4 open on 25' in resp.text def test_backoffice_management_css_class(pub): @@ -340,22 +337,15 @@ def test_backoffice_listing(pub): resp = app.get('/backoffice/management/form-title/') resp.forms['listing-settings']['filter'] = 'all' resp = resp.forms['listing-settings'].submit() - if pub.is_using_postgresql(): - assert resp.text.count('data-link') == 20 - else: - # not using sql -> no pagination - assert resp.text.count('data-link') == 50 + assert resp.text.count('data-link') == 20 # check status filter ') % limit - if get_publisher().is_using_postgresql(): - if order_by is None: - order_by = get_publisher().get_site_option('default-sort-order') or '-receipt_time' - r += htmltext('') % order_by + if order_by is None: + order_by = get_publisher().get_site_option('default-sort-order') or '-receipt_time' + r += htmltext('') % order_by - if get_publisher().is_using_postgresql(): - r += htmltext('

%s

') % self.search_label - if get_request().form.get('q'): - q = force_text(get_request().form.get('q')) - r += htmltext('') % force_str(q) - else: - r += htmltext('') - r += htmltext('') % _('Search') + r += htmltext('

%s

') % self.search_label + if get_request().form.get('q'): + q = force_text(get_request().form.get('q')) + r += htmltext('') % force_str(q) + else: + r += htmltext('') + r += htmltext('') % _('Search') r += self.get_filter_sidebar( selected_filter=selected_filter, @@ -1664,18 +1599,14 @@ class FormPage(Directory): yield FakeField('last_update_time', 'last_update_time', _('Last Modified')) # user fields - if not get_publisher().is_using_postgresql(): - # full name of user, this will load individual user objects to get it - yield FakeField('user-label', 'user-label', _('User Label')) - else: - # user-label field but as a custom field, to get full name of user - # using a sql join clause. - yield UserLabelRelatedField() - for field in get_publisher().user_class.get_fields(): - if not hasattr(field, 'get_view_value'): - continue - field.has_relations = True - yield UserRelatedField(field) + # user-label field but as a custom field, to get full name of user + # using a sql join clause. + yield UserLabelRelatedField() + for field in get_publisher().user_class.get_fields(): + if not hasattr(field, 'get_view_value'): + continue + field.has_relations = True + yield UserRelatedField(field) for field in self.formdef.iter_fields(include_block_fields=True): if getattr(field, 'block_field', None): @@ -1683,8 +1614,6 @@ class FormPage(Directory): # not yet continue yield field - if not get_publisher().is_using_postgresql(): - continue if field.key == 'block': continue if getattr(field, 'block_field', None): @@ -2145,14 +2074,9 @@ class FormPage(Directory): selected_filter_operator = self.get_filter_operator_from_query() criterias = self.get_criterias_from_query() - if get_publisher().is_using_postgresql(): - # only enable pagination in SQL mode, as we do not have sorting in - # the other case. - limit = misc.get_int_or_400( - get_request().form.get('limit', get_publisher().get_site_option('default-page-size') or 20) - ) - else: - limit = misc.get_int_or_400(get_request().form.get('limit', 0)) + limit = misc.get_int_or_400( + get_request().form.get('limit', get_publisher().get_site_option('default-page-size') or 20) + ) offset = misc.get_int_or_400(get_request().form.get('offset', 0)) order_by = misc.get_order_by_or_400(get_request().form.get('order_by')) if self.view and not order_by: @@ -2251,8 +2175,7 @@ class FormPage(Directory): def submit_multi(self, action, selected_filter, selected_filter_operator, query, criterias): item_ids = get_request().form['select[]'] if '_all' in item_ids: - if get_publisher().is_using_postgresql(): - criterias.append(Null('anonymised')) + criterias.append(Null('anonymised')) item_ids = FormDefUI(self.formdef).get_listing_item_ids( selected_filter, selected_filter_operator, @@ -2444,8 +2367,7 @@ class FormPage(Directory): offset=offset, limit=limit, )[0] - if get_publisher().is_using_postgresql(): - self.formdef.data_class().load_all_evolutions(items) + self.formdef.data_class().load_all_evolutions(items) digest_key = 'default' if self.view and isinstance(self.formdef, CardDef): view_digest_key = 'custom-view:%s' % self.view.get_url_slug() @@ -2455,26 +2377,25 @@ class FormPage(Directory): output = [] prefetched_users = None prefetched_roles = None - if get_publisher().is_using_postgresql: - prefetched_users = { - str(x.id): x - for x in get_publisher().user_class.get_ids( - [x.user_id for x in items if x.user_id], ignore_errors=True - ) - if x is not None - } - role_ids = set((self.formdef.workflow_roles or {}).values()) - for filled in items: - if filled.workflow_roles: - for value in filled.workflow_roles.values(): - if not isinstance(value, list): - value = [value] - role_ids |= set(value) - prefetched_roles = { - str(x.id): x - for x in get_publisher().role_class.get_ids(role_ids, ignore_errors=True) - if x is not None - } + prefetched_users = { + str(x.id): x + for x in get_publisher().user_class.get_ids( + [x.user_id for x in items if x.user_id], ignore_errors=True + ) + if x is not None + } + role_ids = set((self.formdef.workflow_roles or {}).values()) + for filled in items: + if filled.workflow_roles: + for value in filled.workflow_roles.values(): + if not isinstance(value, list): + value = [value] + role_ids |= set(value) + prefetched_roles = { + str(x.id): x + for x in get_publisher().role_class.get_ids(role_ids, ignore_errors=True) + if x is not None + } for filled in items: data = filled.get_json_export_dict( include_files=False, @@ -2717,7 +2638,6 @@ class FormPage(Directory): get_response().filter['sidebar'] = self.get_formdata_sidebar() + self.get_stats_sidebar( selected_filter ) - do_graphs = get_publisher().is_using_postgresql() displayed_criterias = None if selected_filter and selected_filter != 'all': @@ -2749,10 +2669,9 @@ class FormPage(Directory): criterias = [StrictNotEqual('status', 'draft')] + displayed_criterias values = self.formdef.data_class().select(criterias) - if get_publisher().is_using_postgresql(): - # load all evolutions in a single batch, to avoid as many query as - # there are formdata when computing resolution times statistics. - self.formdef.data_class().load_all_evolutions(values) + # load all evolutions in a single batch, to avoid as many query as + # there are formdata when computing resolution times statistics. + self.formdef.data_class().load_all_evolutions(values) r += htmltext('
') if displayed_criterias: @@ -2764,8 +2683,7 @@ class FormPage(Directory): if criteria_label: r += htmltext('
  • %s
  • ') % criteria_label r += htmltext('
    ') - if do_graphs: - r += htmltext('
    ') + r += htmltext('
    ') no_forms = len(values) r += htmltext('
    ') @@ -2799,12 +2717,11 @@ class FormPage(Directory): r += stats_times r += htmltext('
    ') - if do_graphs: - r += htmltext('
    ') - r += htmltext('
    ') - criterias.append(Equal('formdef_id', int(self.formdef.id))) - r += do_graphs_section(criterias=criterias) - r += htmltext('
    ') + r += htmltext('
    ') + r += htmltext('
    ') + criterias.append(Equal('formdef_id', int(self.formdef.id))) + r += do_graphs_section(criterias=criterias) + r += htmltext('
    ') r += htmltext('') # id="statistics" @@ -3165,11 +3082,7 @@ class FormBackOfficeStatusPage(FormStatusPage): if formdata.formdef.lateral_template: r += htmltext('
    ' % formdata.get_url(backoffice=True)) - if ( - not isinstance(formdata.formdef, CardDef) - and formdata.user_id - and get_publisher().is_using_postgresql() - ): + if not isinstance(formdata.formdef, CardDef) and formdata.user_id: r += htmltext( '
    ' % formdata.get_url(backoffice=True) ) diff --git a/wcs/backoffice/submission.py b/wcs/backoffice/submission.py index 7440f7b2f..30878a2f5 100644 --- a/wcs/backoffice/submission.py +++ b/wcs/backoffice/submission.py @@ -202,15 +202,9 @@ class FormFillPage(PublicFormFillPage): return data_class = self.formdef.data_class() - if get_publisher().is_using_postgresql(): - has = bool( - data_class.count([StrictNotEqual('status', 'draft'), Equal('user_id', formdata.user_id)]) - ) - else: - has = any( - x for x in data_class.get_with_indexed_value('user_id', formdata.user_id) if not x.is_draft() - ) - context['user_has_already_one_such_form'] = has + context['user_has_already_one_such_form'] = bool( + data_class.count([StrictNotEqual('status', 'draft'), Equal('user_id', formdata.user_id)]) + ) def get_sidebar(self, data): r = TemplateIO(html=True) diff --git a/wcs/carddef.py b/wcs/carddef.py index 2f534e5c9..b492ab15b 100644 --- a/wcs/carddef.py +++ b/wcs/carddef.py @@ -60,7 +60,7 @@ class CardDef(FormDef): # carddef if data_class._formdef is self: return data_class - if (get_publisher().is_using_postgresql() and not mode == 'files') or mode == 'sql': + if (not mode == 'files') or mode == 'sql': from . import sql table_name = sql.get_formdef_table_name(self) diff --git a/wcs/ctl/delete_tenant.py b/wcs/ctl/delete_tenant.py index b30129829..e3065118c 100644 --- a/wcs/ctl/delete_tenant.py +++ b/wcs/ctl/delete_tenant.py @@ -51,70 +51,66 @@ class CmdDeleteTenant(Command): deletion_date = datetime.now().strftime('%Y%m%d_%H%M%S_%f') os.rename(pub.app_dir, pub.app_dir + '_removed_%s.invalid' % deletion_date) - # do this only if the wcs has a postgresql configuration - if pub.is_using_postgresql(): - postgresql_cfg = {} - for k, v in pub.cfg['postgresql'].items(): - if v and isinstance(v, str): - postgresql_cfg[k] = v + postgresql_cfg = {} + for k, v in pub.cfg['postgresql'].items(): + if v and isinstance(v, str): + postgresql_cfg[k] = v - # if there's a createdb-connection-params, we can do a DROP DATABASE with - # the option --force-drop, rename it if not - createdb_cfg = pub.cfg['postgresql'].get('createdb-connection-params', {}) - createdb = True - if not createdb_cfg: - createdb_cfg = postgresql_cfg - createdb = False - if 'database' in createdb_cfg: - createdb_cfg['dbname'] = createdb_cfg.pop('database') - try: - pgconn = psycopg2.connect(**createdb_cfg) - except psycopg2.Error as e: - print( - 'failed to connect to postgresql (%s)' % psycopg2.errorcodes.lookup(e.pgcode), - file=sys.stderr, - ) - return + # if there's a createdb-connection-params, we can do a DROP DATABASE with + # the option --force-drop, rename it if not + createdb_cfg = pub.cfg['postgresql'].get('createdb-connection-params', {}) + createdb = True + if not createdb_cfg: + createdb_cfg = postgresql_cfg + createdb = False + if 'database' in createdb_cfg: + createdb_cfg['dbname'] = createdb_cfg.pop('database') + try: + pgconn = psycopg2.connect(**createdb_cfg) + except psycopg2.Error as e: + print( + 'failed to connect to postgresql (%s)' % psycopg2.errorcodes.lookup(e.pgcode), + file=sys.stderr, + ) + return - pgconn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - cur = pgconn.cursor() - dbname = postgresql_cfg.get('dbname') or postgresql_cfg.get('database') - try: - if createdb: - if options.force_drop: - cur.execute('DROP DATABASE %s' % dbname) - else: - cur.execute( - 'ALTER DATABASE %s RENAME TO removed_%s_%s' % (dbname, deletion_date, dbname) - ) + pgconn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = pgconn.cursor() + dbname = postgresql_cfg.get('dbname') or postgresql_cfg.get('database') + try: + if createdb: + if options.force_drop: + cur.execute('DROP DATABASE %s' % dbname) else: - cur.execute( - """SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - AND table_type = 'BASE TABLE'""" - ) - - tables_names = [x[0] for x in cur.fetchall()] - - if options.force_drop: - for table_name in tables_names: - cur.execute('DROP TABLE %s CASCADE' % table_name) - - else: - schema_name = 'removed_%s_%s' % (deletion_date, dbname) - cur.execute("CREATE SCHEMA %s" % schema_name[:63]) - for table_name in tables_names: - cur.execute('ALTER TABLE %s SET SCHEMA %s' % (table_name, schema_name[:63])) - - except psycopg2.Error as e: - print( - 'failed to alter database %s: (%s)' % (dbname, psycopg2.errorcodes.lookup(e.pgcode)), - file=sys.stderr, + cur.execute('ALTER DATABASE %s RENAME TO removed_%s_%s' % (dbname, deletion_date, dbname)) + else: + cur.execute( + """SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE'""" ) - return - cur.close() + tables_names = [x[0] for x in cur.fetchall()] + + if options.force_drop: + for table_name in tables_names: + cur.execute('DROP TABLE %s CASCADE' % table_name) + + else: + schema_name = 'removed_%s_%s' % (deletion_date, dbname) + cur.execute("CREATE SCHEMA %s" % schema_name[:63]) + for table_name in tables_names: + cur.execute('ALTER TABLE %s SET SCHEMA %s' % (table_name, schema_name[:63])) + + except psycopg2.Error as e: + print( + 'failed to alter database %s: (%s)' % (dbname, psycopg2.errorcodes.lookup(e.pgcode)), + file=sys.stderr, + ) + return + + cur.close() CmdDeleteTenant.register() diff --git a/wcs/ctl/rebuild_indexes.py b/wcs/ctl/rebuild_indexes.py index ba34ca684..69a6e4e33 100644 --- a/wcs/ctl/rebuild_indexes.py +++ b/wcs/ctl/rebuild_indexes.py @@ -28,18 +28,6 @@ def rebuild_vhost_indexes(pub, destroy=False): if destroy: Role.destroy_indexes() Role.rebuild_indexes() - if pub.is_using_postgresql(): - return - from wcs.users import User - - if destroy: - User.destroy_indexes() - User.rebuild_indexes() - for formdef in FormDef.select(): - formdata = formdef.data_class() - if destroy: - formdata.destroy_indexes() - formdata.rebuild_indexes() class CmdRebuildIndexes(Command): diff --git a/wcs/formdata.py b/wcs/formdata.py index 581075017..dbbdecc72 100644 --- a/wcs/formdata.py +++ b/wcs/formdata.py @@ -424,16 +424,13 @@ class FormData(StorableObject): @classmethod def get_actionable_count(cls, user_roles): - if get_publisher().is_using_postgresql(): - statuses = ['wf-%s' % x.id for x in cls._formdef.workflow.get_not_endpoint_status()] - criterias = [ - Intersects('actions_roles_array', user_roles), - Contains('status', statuses), - Null('anonymised'), - ] - return cls.count(criterias) - else: - return len(cls.get_actionable_ids(user_roles)) + statuses = ['wf-%s' % x.id for x in cls._formdef.workflow.get_not_endpoint_status()] + criterias = [ + Intersects('actions_roles_array', user_roles), + Contains('status', statuses), + Null('anonymised'), + ] + return cls.count(criterias) @classmethod def get_actionable_ids_criteria(cls, user_roles): @@ -442,17 +439,7 @@ class FormData(StorableObject): @classmethod def get_actionable_ids(cls, user_roles): - if get_publisher().is_using_postgresql(): - return cls.keys([cls.get_actionable_ids_criteria(user_roles)]) - else: - statuses = ['wf-%s' % x.id for x in cls._formdef.workflow.get_not_endpoint_status()] - actions_ids = set() - for role in user_roles: - actions_ids |= set(cls.get_ids_with_indexed_value('actions_roles', str(role))) - open_ids = [] - for status_id in statuses: - open_ids.extend(cls.get_ids_with_indexed_value('status', status_id)) - return list(actions_ids.intersection(open_ids)) + return cls.keys([cls.get_actionable_ids_criteria(user_roles)]) @classmethod def get_submission_channels(cls): diff --git a/wcs/formdef.py b/wcs/formdef.py index dbbd12cb7..be9e15e90 100644 --- a/wcs/formdef.py +++ b/wcs/formdef.py @@ -44,7 +44,7 @@ from .qommon.cron import CronJob from .qommon.form import Form, HtmlWidget, UploadedFile from .qommon.misc import JSONEncoder, get_as_datetime, is_attachment, is_upload, simplify, xml_node_text from .qommon.publisher import get_publisher_class -from .qommon.storage import Equal, StorableObject, StrictNotEqual, fix_key +from .qommon.storage import Equal, StorableObject, fix_key from .qommon.substitution import Substitutions from .qommon.template import Template from .roles import logged_users_role @@ -329,7 +329,7 @@ class FormDef(StorableObject): @classmethod def remove_object(cls, id): super().remove_object(id) - if get_publisher().is_using_postgresql() and cls == FormDef: + if cls == FormDef: # recreate global views so they don't reference formdata from # deleted formefs from . import sql @@ -352,7 +352,7 @@ class FormDef(StorableObject): # formdef if data_class._formdef is self: return data_class - if (get_publisher().is_using_postgresql() and not mode == 'files') or mode == 'sql': + if (not mode == 'files') or mode == 'sql': from . import sql table_name = sql.get_formdef_table_name(self) @@ -411,8 +411,7 @@ class FormDef(StorableObject): @classmethod def get_new_id(cls, create=False): id = super().get_new_id(create=False) - if get_publisher().is_using_postgresql(): - id = cls.get_sql_new_id(id_start=int(id)) + id = cls.get_sql_new_id(id_start=int(id)) if create: objects_dir = cls.get_objects_dir() object_filename = os.path.join(objects_dir, fix_key(id)) @@ -443,8 +442,7 @@ class FormDef(StorableObject): @classmethod def wipe(cls): super().wipe() - if get_publisher().is_using_postgresql(): - cls.sql_wipe() + cls.sql_wipe() @classmethod def sql_wipe(cls): @@ -457,6 +455,9 @@ class FormDef(StorableObject): if self.url_name is None: # set url name if it's not yet there self.url_name = self.get_new_url_name() + + object_only = kwargs.pop('object_only', False) + new_internal_identifier = self.get_new_internal_identifier() if not self.internal_identifier: self.internal_identifier = new_internal_identifier @@ -466,9 +467,8 @@ class FormDef(StorableObject): # or if there are not yet any submitted forms (or if site # is using the SQL storage as internal identifier is not used # in that mode. - if self.id is None or get_publisher().is_using_postgresql() or not (self.data_class().exists()): + if self.id is None or not self.data_class().exists(): self.internal_identifier = new_internal_identifier - object_only = kwargs.pop('object_only', False) if not object_only: self.update_relations() @@ -487,9 +487,6 @@ class FormDef(StorableObject): self.store_related_custom_views() def update_storage(self): - if not get_publisher().is_using_postgresql(): - return - from . import sql actions = sql.do_formdef_tables(self, rebuild_views=True, rebuild_global_views=True) @@ -1919,7 +1916,6 @@ class FormDef(StorableObject): status.id in status_mapping for status in old_workflow.possible_status ), 'a status was not mapped' - unmapped_status_suffix = '-invalid-%s' % str(self.workflow_id or 'default') mapping = {} for old_status, new_status in status_mapping.items(): mapping['wf-%s' % old_status] = 'wf-%s' % new_status @@ -1927,28 +1923,9 @@ class FormDef(StorableObject): if any(x[0] != x[1] for x in mapping.items()): # if there are status changes, update all formdatas (except drafts) - if get_publisher().is_using_postgresql(): - from . import sql + from . import sql - sql.formdef_remap_statuses(self, mapping) - else: - - def map_status(status): - if status is None: - return None - elif status in mapping: - return mapping[status] - elif '-invalid-' in status: - return status - else: - return '%s%s' % (status, unmapped_status_suffix) - - for formdata in self.data_class().select([StrictNotEqual('status', 'draft')]): - formdata.status = map_status(formdata.status) - if formdata.evolution: - for evo in formdata.evolution: - evo.status = map_status(evo.status) - formdata.store() + sql.formdef_remap_statuses(self, mapping) self.workflow = new_workflow if new_workflow.has_action('geolocate') and not self.geolocations: diff --git a/wcs/forms/backoffice.py b/wcs/forms/backoffice.py index c0f827118..e7673fa64 100644 --- a/wcs/forms/backoffice.py +++ b/wcs/forms/backoffice.py @@ -46,15 +46,12 @@ class FormDefUI: ): # noqa pylint: disable=too-many-arguments - using_postgresql = get_publisher().is_using_postgresql() - if not items: if offset and not limit: limit = int(get_publisher().get_site_option('default-page-size') or 20) if not criterias: criterias = [] - if using_postgresql: - criterias.append(Null('anonymised')) + criterias.append(Null('anonymised')) items, total_count = self.get_listing_items( fields, selected_filter, @@ -101,7 +98,7 @@ class FormDefUI: r += htmltext('') r += htmltext('') - if self.formdef.workflow.criticality_levels and using_postgresql: + if self.formdef.workflow.criticality_levels: r += htmltext( '' ) @@ -127,7 +124,7 @@ class FormDefUI: else: field_sort_key = 'f%s' % f.contextual_id - if field_sort_key and using_postgresql: + if field_sort_key: r += htmltext('') % field_sort_key else: r += htmltext('') @@ -144,9 +141,8 @@ class FormDefUI: r += htmltext('') r += htmltext('') - if get_publisher().is_using_postgresql(): - # add links to paginate - r += pagination_links(offset, limit, total_count) + # add links to paginate + r += pagination_links(offset, limit, total_count) return r.getvalue() @@ -159,96 +155,6 @@ class FormDefUI: user=None, criterias=None, anonymise=False, - ): - if get_publisher().is_using_postgresql(): - return self.get_listing_item_ids_sql( - selected_filter, selected_filter_operator, query, order_by, user, criterias, anonymise - ) - - def get_all_except_drafts(): - item_ids = formdata_class.keys() - drafts = formdata_class.get_ids_with_indexed_value('status', 'draft') - return [x for x in item_ids if x not in drafts] - - formdata_class = self.formdef.data_class() - # used for postgresql mode only - if selected_filter == 'all': - if selected_filter_operator == 'ne': - # nothing - item_ids = [] - else: - # all except drafts - item_ids = get_all_except_drafts() - elif selected_filter == 'waiting': - user_roles = [logged_users_role().id] + user.get_roles() - waiting_item_ids = formdata_class.get_actionable_ids(user_roles) - if selected_filter_operator == 'ne': - # select all ids except drafts - item_ids = get_all_except_drafts() - # exclude waiting ids - item_ids = [x for x in item_ids if x not in waiting_item_ids] - else: - # only waiting ids - item_ids = waiting_item_ids - else: - # build selected status list - applied_filters = [] - if selected_filter == 'pending': - applied_filters = ['wf-%s' % x.id for x in self.formdef.workflow.get_not_endpoint_status()] - elif selected_filter == 'done': - applied_filters = ['wf-%s' % x.id for x in self.formdef.workflow.get_endpoint_status()] - else: - applied_filters = ['wf-%s' % selected_filter] - - filtered_item_ids = [] - for status_id in applied_filters: - filtered_item_ids.extend( - formdata_class.get_ids_with_indexed_value( - 'status', - status_id, - ) - ) - if selected_filter_operator == 'ne': - # select all ids except drafts - item_ids = get_all_except_drafts() - # exclude selected status list - item_ids = [x for x in item_ids if x not in filtered_item_ids] - else: - # only selected status list - item_ids = filtered_item_ids - - if query: - query_ids = formdata_class.get_ids_from_query(query) - item_ids = list(set(item_ids).intersection(query_ids)) - - if criterias: - select_ids = formdata_class.keys(clause=criterias) - item_ids = list(set(item_ids).intersection(select_ids)) - - if item_ids and not anonymise: - # as we are in the backoffice, we don't have to care about the - # situation where the user is the submitter, and we limit ourselves - # to consider treating roles. - if not user.is_admin: - user_roles = set(user.get_roles()) - concerned_ids = set() - for role in user_roles: - concerned_ids |= set( - formdata_class.get_ids_with_indexed_value('concerned_roles', str(role)) - ) - item_ids = list(set(item_ids).intersection(concerned_ids)) - - return item_ids - - def get_listing_item_ids_sql( - self, - selected_filter, - selected_filter_operator, - query, - order_by, - user, - criterias, - anonymise, ): formdata_class = self.formdef.data_class() criterias = [] or criterias[:] @@ -354,9 +260,6 @@ class FormDefUI: criterias=criterias, anonymise=anonymise, ) - if not get_publisher().is_using_postgresql(): - item_ids.sort(key=int) - item_ids.reverse() total_count = len(item_ids) @@ -364,8 +267,7 @@ class FormDefUI: offset = 0 kwargs = {} - if get_publisher().is_using_postgresql(): - kwargs['fields'] = fields + kwargs['fields'] = fields if limit: items = formdata_class.get_ids(item_ids[offset : offset + limit], keep_order=True, **kwargs) diff --git a/wcs/publisher.py b/wcs/publisher.py index d3e25e6e1..28742dfa1 100644 --- a/wcs/publisher.py +++ b/wcs/publisher.py @@ -132,9 +132,6 @@ class WcsPublisher(QommonPublisher): DeprecationsScanAfterJob().execute() - def is_using_postgresql(self): - return True - def has_postgresql_config(self): return bool(self.cfg.get('postgresql', {})) @@ -467,10 +464,9 @@ class WcsPublisher(QommonPublisher): def cleanup(self): self._cached_user_fields_formdef = None - if self.is_using_postgresql(): - from . import sql + from . import sql - sql.cleanup_connection() + sql.cleanup_connection() @contextmanager def complex_data(self): diff --git a/wcs/qommon/cron.py b/wcs/qommon/cron.py index 90a227666..25686cdc2 100644 --- a/wcs/qommon/cron.py +++ b/wcs/qommon/cron.py @@ -84,8 +84,7 @@ def cron_worker(publisher, now, job_name=None, delayed_jobs=None): jobs = delayed_jobs else: # reindex user and formdata if needed (should only be run once) - if publisher.is_using_postgresql(): - publisher.reindex_sql() + publisher.reindex_sql() jobs = [] diff --git a/wcs/qommon/tokens.py b/wcs/qommon/tokens.py index e059901b9..27c31466c 100644 --- a/wcs/qommon/tokens.py +++ b/wcs/qommon/tokens.py @@ -19,7 +19,6 @@ import random import string from django.utils.timezone import make_aware, now -from quixote import get_publisher from .storage import Equal, Less, StorableObject @@ -75,16 +74,5 @@ class Token(StorableObject): @classmethod def clean(cls): - if get_publisher().is_using_postgresql(): - # noqa pylint: disable=unexpected-keyword-arg - cls.wipe(clause=[Less('expiration', now())]) - else: - for token_id in cls.keys(): - try: - cls.get(token_id) # will run migrate, will check expiration - except KeyError: - pass - except AttributeError: - # old python2 tokens: - # AttributeError: module 'builtins' has no attribute 'unicode' - cls.remove_object(token_id) + # noqa pylint: disable=unexpected-keyword-arg + cls.wipe(clause=[Less('expiration', now())]) diff --git a/wcs/sql.py b/wcs/sql.py index 6d9a611da..03df9ba60 100644 --- a/wcs/sql.py +++ b/wcs/sql.py @@ -1581,9 +1581,6 @@ def do_meta_table(conn=None, cur=None, insert_current_sql_level=True): @guard_postgres def redo_views(conn, cur, formdef, rebuild_global_views=False): - if get_publisher().get_site_option('postgresql_views') == 'false': - return - if formdef.id is None: return @@ -1961,9 +1958,13 @@ class SqlMixin: sql_statement += ' WHERE ' + ' AND '.join(where_clauses) sql_statement += ' LIMIT 1' conn, cur = get_connection_and_cursor() - cur.execute(sql_statement, parameters) - check = cur.fetchone() - result = check is not None + try: + cur.execute(sql_statement, parameters) + except psycopg2.errors.UndefinedTable: + result = False + else: + check = cur.fetchone() + result = check is not None conn.commit() cur.close() return result diff --git a/wcs/statistics/views.py b/wcs/statistics/views.py index 8f9912f0c..29eaef8a7 100644 --- a/wcs/statistics/views.py +++ b/wcs/statistics/views.py @@ -19,7 +19,6 @@ import collections from django.http import HttpResponseBadRequest, HttpResponseForbidden, JsonResponse from django.urls import reverse from django.views.generic import View -from quixote import get_publisher from wcs import sql from wcs.api_utils import is_url_signed @@ -42,9 +41,6 @@ class RestrictedView(View): class IndexView(RestrictedView): def get(self, request, *args, **kwargs): - if not get_publisher().is_using_postgresql(): - return JsonResponse({'data': [], 'err': 0}) - categories = Category.select() categories.sort(key=lambda x: misc.simplify(x.name)) category_options = [{'id': '_all', 'label': C_('categories|All')}] + [ @@ -306,8 +302,6 @@ class FormsCountView(RestrictedView): for status in waitpoint_status: options.append((status.id, status.name)) elif field.type in ('item', 'items'): - if not get_publisher().is_using_postgresql(): - continue options = form_page.get_item_filter_options(field, selected_filter='all', anonymised=True) if not options: continue diff --git a/wcs/users.py b/wcs/users.py index 00ab922be..176b3aabb 100644 --- a/wcs/users.py +++ b/wcs/users.py @@ -188,9 +188,6 @@ class User(StorableObject): @classmethod def get_users_with_roles(cls, included_roles=None, excluded_roles=None, order_by=None): - if not get_publisher().is_using_postgresql(): - return [] - from wcs import sql criterias = [sql.Null('deleted_timestamp')] diff --git a/wcs/variables.py b/wcs/variables.py index 4a60565db..947ab7898 100644 --- a/wcs/variables.py +++ b/wcs/variables.py @@ -69,10 +69,7 @@ class LazyFormDefObjectsManager: ) def order_by(self, attribute): - if get_publisher().is_using_postgresql(): - field = self.get_field(attribute) - else: - field = None # no support for field search + field = self.get_field(attribute) return self._clone(self._criterias, order_by=field or attribute) def limit(self, limit): diff --git a/wcs/wf/jump.py b/wcs/wf/jump.py index bc1e8b5d7..624dd458b 100644 --- a/wcs/wf/jump.py +++ b/wcs/wf/jump.py @@ -340,28 +340,25 @@ def _apply_timeouts(publisher, **kwargs): ) if job else contextlib.ExitStack(): formdata_class = formdef.data_class() for status_id in status_ids: - if publisher.is_using_postgresql(): - # get minimum delay for jumps in this status - delay = math.inf - for jump_action in wfs_status[str(formdef.workflow_id)][status_id]: - if Template.is_template_string(jump_action.timeout): - delay = 0 - break - delay = min(delay, int(jump_action.timeout)) - # limit delay to minimal delay - if delay < JUMP_TIMEOUT_INTERVAL * 60: - delay = JUMP_TIMEOUT_INTERVAL * 60 + # get minimum delay for jumps in this status + delay = math.inf + for jump_action in wfs_status[str(formdef.workflow_id)][status_id]: + if Template.is_template_string(jump_action.timeout): + delay = 0 + break + delay = min(delay, int(jump_action.timeout)) + # limit delay to minimal delay + if delay < JUMP_TIMEOUT_INTERVAL * 60: + delay = JUMP_TIMEOUT_INTERVAL * 60 - criterias = [ - Equal('status', status_id), - LessOrEqual( - 'last_update_time', - (datetime.datetime.now() - datetime.timedelta(seconds=delay)).timetuple(), - ), - ] - formdatas = formdata_class.select_iterator(criterias, ignore_errors=True, itersize=200) - else: - formdatas = formdata_class.get_with_indexed_value('status', status_id, ignore_errors=True) + criterias = [ + Equal('status', status_id), + LessOrEqual( + 'last_update_time', + (datetime.datetime.now() - datetime.timedelta(seconds=delay)).timetuple(), + ), + ] + formdatas = formdata_class.select_iterator(criterias, ignore_errors=True, itersize=200) for formdata in formdatas: for jump_action in wfs_status[str(formdef.workflow_id)][formdata.status]: