tests/admin_pages/test_form.py

@@ -5164,3 +5164,24 @@ def test_forms_last_test_result(pub, formdef):
     TestDef.remove_object(testdef.id)
     resp = app.get('/backoffice/forms/1/')
     assert 'Last tests run' not in resp.text
+
+
+def test_admin_form_sql_integrity_error(pub):
+    create_superuser(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [fields.BoolField(id='1', label='Bool')]
+    formdef.store()
+
+    formdef.fields = [fields.StringField(id='1', label='String')]
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get(formdef.get_admin_url())
+    assert (
+        resp.pyquery('.errornotice summary').text()
+        == 'There are integrity errors in the database column types.'
+    )
+    assert resp.pyquery('.errornotice li').text() == 'String, expected: character varying, got: boolean.'

tests/test_formdef.py

@@ -647,7 +647,7 @@ def test_wipe_on_object(pub):
         formdef.wipe()
 
 
-def test_update_storage_all_formdefs(pub):
+def test_update_storage_all_formdefs(pub, capfd):
     CardDef.wipe()
     FormDef.wipe()
 
@@ -664,6 +664,18 @@ def test_update_storage_all_formdefs(pub):
         update_storage_all_formdefs(pub)
         assert update_storage.call_count == 10
 
+    assert not capfd.readouterr().out
+
+    formdef = FormDef()
+    formdef.name = 'broken formdef'
+    formdef.fields = [StringField(id='1', label='Test')]
+    formdef.store()
+    formdef.fields = [DateField(id='1', label='Test')]
+    formdef.store()
+
+    update_storage_all_formdefs(pub)
+    assert capfd.readouterr().out == '! Integrity errors in %s\n' % formdef.get_admin_url()
+
 
 def test_lazy_formdef(pub):
     FormDef.wipe()

tests/test_sql.py

@@ -2983,3 +2983,20 @@ def test_sql_data_views(pub_with_views, formdef_class):
     assert column_exists_in_table(cur, f'{prefix}_test', 'geoloc_base_x')
     conn.commit()
     cur.close()
+
+
+def test_sql_integrity_errors(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.StringField(id='1', label='string'),
+    ]
+    formdef.store()
+    assert not formdef.sql_integrity_errors
+
+    formdef.fields = [
+        fields.FileField(id='1', label='string'),
+    ]
+    formdef.store()
+    assert formdef.sql_integrity_errors == {'1': {'got': 'character varying', 'expected': 'bytea'}}

wcs/formdef.py

@@ -189,6 +189,7 @@ class FormDef(StorableObject):
 
     geolocations = None
     history_pane_default_mode = 'expanded'
+    sql_integrity_errors = None
 
     # store reverse relations
     reverse_relations = None
@@ -587,6 +588,9 @@ class FormDef(StorableObject):
     def get_all_fields(self):
         return (self.fields or []) + self.workflow.get_backoffice_fields()
 
+    def get_all_fields_dict(self):
+        return {x.id: x for x in self.get_all_fields()}
+
     def get_total_count_data_fields(self):
         count = len([x for x in self.fields or [] if not x.is_no_data_field and not x.key == 'block'])
         for field in self.fields or []:
@@ -2334,6 +2338,10 @@ def update_storage_all_formdefs(publisher, **kwargs):
 
     for formdef in itertools.chain(FormDef.select(), CardDef.select()):
         formdef.update_storage()
+        if formdef.sql_integrity_errors:
+            # print errors, this will get them in the cron output, that hopefully
+            # a sysadmin will read.
+            print(f'! Integrity errors in {formdef.get_admin_url()}')
 
 
 def get_formdefs_of_all_kinds(**kwargs):

wcs/sql.py

@@ -544,6 +544,7 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
             cur.execute(f'ALTER TABLE {table_name} ALTER COLUMN last_update_time SET DATA TYPE timestamptz')
 
         # add new fields
+        field_integrity_errors = {}
         for field in formdef.get_all_fields():
             assert field.id is not None
             sql_type = SQL_TYPE_MAPPING.get(field.key, 'varchar')
@@ -554,6 +555,16 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
                 cur.execute(
                     '''ALTER TABLE %s ADD COLUMN %s %s''' % (table_name, get_field_id(field), sql_type)
                 )
+            else:
+                existing_type = existing_field_types.get(get_field_id(field))
+                # map to names returned in data_type column
+                expected_type = {
+                    'varchar': 'character varying',
+                    'text[]': 'ARRAY',
+                    'text[][]': 'ARRAY',
+                }.get(sql_type) or sql_type
+                if existing_type != expected_type:
+                    field_integrity_errors[str(field.id)] = {'got': existing_type, 'expected': expected_type}
             if field.store_display_value:
                 needed_fields.add('%s_display' % get_field_id(field))
                 if '%s_display' % get_field_id(field) not in existing_fields:
@@ -569,6 +580,10 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
                         % (table_name, '%s_structured' % get_field_id(field))
                     )
 
+        if (field_integrity_errors or None) != formdef.sql_integrity_errors:
+            formdef.sql_integrity_errors = field_integrity_errors
+            formdef.store(object_only=True)
+
         for field in (formdef.geolocations or {}).keys():
             column_name = 'geoloc_%s' % field
             needed_fields.add(column_name)

wcs/templates/wcs/backoffice/carddef.html

@@ -14,7 +14,9 @@
       </p>
     {% endif %}
   </div>
+
   {{ publisher.get_request.session.display_message|safe }}
+  {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>

wcs/templates/wcs/backoffice/formdef.html

@@ -35,6 +35,7 @@
   </div>
 
   {{ publisher.get_request.session.display_message|safe }}
+  {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>

wcs/templates/wcs/backoffice/includes/sql-fields-integrity.html

@@ -0,0 +1,25 @@
+{% load i18n %}
+
+{% if formdef.sql_integrity_errors %}
+  <div class="errornotice">
+    <details><summary>
+      {% blocktrans trimmed %}
+        There are integrity errors in the database column types.
+      {% endblocktrans %}
+    </summary>
+      <ul>
+        {% for error in formdef.sql_integrity_errors.items %}
+          <li>
+            {% with field=formdef.get_all_fields_dict|get:error.0 %}
+              <a href="{{ field.get_admin_url }}">{{ field.ellipsized_label }}</a>,
+              {% blocktrans trimmed with expected=error.1.expected got=error.1.got %}
+                expected: {{ expected }}, got: {{ got }}.
+              {% endblocktrans %}
+            {% endwith %}
+          </li>
+        {% endfor %}
+      </ul>
+    </p>
+  </details>
+  </div>
+{% endif %}