general: update convert_to_sql for postgresql enabled by default (#67190)

tests/test_convert_to_sql.py

@@ -18,14 +18,16 @@ from .utilities import clean_temporary_pub, create_temporary_pub, force_connecti
 
 @pytest.fixture
 def formdeffix():
-    FormDef.wipe()
+    for formdef in FormDef.select():
+        formdef.remove_self()
     formdef = FormDef()
+    formdef.id = 123
     formdef.name = 'testform'
     formdef.description = 'plop'
     formdef.fields = [BoolField(id='1')]
     formdef.store()
 
-    data_class = formdef.data_class()
+    data_class = formdef.data_class(mode='files')
     for value in (True, True, True, False):
         formdata = data_class()
         formdata.data = {'1': value}
@@ -36,14 +38,16 @@ def formdeffix():
 
 @pytest.fixture
 def carddeffix():
-    CardDef.wipe()
+    for carddef in CardDef.select():
+        carddef.remove_self()
     carddef = CardDef()
+    carddef.id = 456
     carddef.name = 'testcard'
     carddef.description = 'plop'
     carddef.fields = [BoolField(id='1')]
     carddef.store()
 
-    data_class = carddef.data_class()
+    data_class = carddef.data_class(mode='files')
     for value in (True, True, True, False):
         formdata = data_class()
         formdata.data = {'1': value}
@@ -144,12 +148,7 @@ def test_migration(pub, database):
 
 
 def test_data_is_migrated(pub, database, local_user, formdeffix, carddeffix):
-    pub.load_site_options()
-    assert not pub.site_options.has_option('options', 'postgresql')
     call_command('convert_to_sql', '-d', 'example.net', '--database', database)
-    pub.load_site_options()
-    assert pub.site_options.has_option('options', 'postgresql')
-    pub.is_using_postgresql = lambda: True
     pub.set_config()
     formdefs = FormDef.select()
     assert len(formdefs) == 1
@@ -177,12 +176,6 @@ def test_users_and_roles(pub, database, local_user):
     role = Role(name='Test Role')
     role.store()
 
-    pub.load_site_options()
-    assert not pub.site_options.has_option('options', 'postgresql')
     call_command('convert_to_sql', '-d', 'example.net', '--database', database)
-    pub.load_site_options()
-    assert pub.site_options.has_option('options', 'postgresql')
-    pub.is_using_postgresql = lambda: True
-    pub.set_config()
     assert len(pub.user_class.get_users_with_name_identifier('0123456789')) == 1
     assert pub.role_class.count() == 1

wcs/ctl/management/commands/convert_to_sql.py

@@ -14,21 +14,17 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
-import io
-import os
 import sys
 import traceback
 
 import psycopg2
 from django.core.management.base import BaseCommand, CommandError
-from django.utils.encoding import force_bytes
 
 from wcs import sql
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.misc import localstrftime
 from wcs.qommon.publisher import UnknownTenantError, get_publisher_class
-from wcs.qommon.storage import atomic_write
 from wcs.roles import Role
 from wcs.users import User
 
@@ -47,16 +43,20 @@ class Command(BaseCommand):
 
     def handle(self, **options):
         self.publisher = self.get_publisher(options['domain'])
-        if self.publisher.is_using_postgresql():
+        if self.publisher.has_postgresql_config():
             raise CommandError('tenant already using postgresql')
 
         self.setup_connection(**options)
         sql.get_connection(new=True)
+
+        # create early tables
+        sql.do_snapshots_table()
+        sql.do_custom_views_table()
+
         self.store_roles()
         self.store_users()
         self.store_forms()
         self.publisher.write_cfg()
-        self.enable_connection()
         self.publisher.initialize_sql()
         self.publisher.cleanup()
 
@@ -76,16 +76,6 @@ class Command(BaseCommand):
                 options[k] = kwargs.get(k)
         self.publisher.cfg['postgresql'] = options
 
-    def enable_connection(self):
-        if not self.publisher.site_options.has_option('options', 'postgresql'):
-            if not self.publisher.site_options.has_section('options'):
-                self.publisher.site_options.add_section('options')
-            self.publisher.site_options.set('options', 'postgresql', 'true')
-            options_file = os.path.join(self.publisher.app_dir, 'site-options.cfg')
-            stringio = io.StringIO()
-            self.publisher.site_options.write(stringio)
-            atomic_write(options_file, force_bytes(stringio.getvalue()))
-
     def store_users(self):
         self.convert_objects('user', User, sql.SqlUser, skip_global_forms_table_update=True)
         sql.SqlUser.fix_sequences()
@@ -118,7 +108,7 @@ class Command(BaseCommand):
 
     def store_forms(self):
         errors = []
-        for formdef in FormDef.select() + CardDef.select():
+        for formdef in FormDef.select(ignore_migration=True) + CardDef.select(ignore_migration=True):
             print('converting %s' % formdef.name)
             sql.do_formdef_tables(formdef, rebuild_views=True, rebuild_global_views=True)
             data_class = formdef.data_class(mode='files')
@@ -160,6 +150,10 @@ class Command(BaseCommand):
                 self.update_progress(100 * i / count)
             sql_data_class.fix_sequences()
 
+            if formdef.migrate():
+                # run formdef migration as we're done and table name can be stored etc.
+                formdef.store()
+
         if errors:
             with open('error_formdata.log', 'w') as error_log:
                 for formdata, trace in errors:

wcs/formdef.py

@@ -482,8 +482,9 @@ class FormDef(StorableObject):
                 instance=self, comment=comment, store_user=snapshot_store_user
             )
 
-        self.update_storage()
-        self.store_related_custom_views()
+        if get_publisher().has_postgresql_config():
+            self.update_storage()
+            self.store_related_custom_views()
 
     def update_storage(self):
         if not get_publisher().is_using_postgresql():

wcs/publisher.py

@@ -154,7 +154,7 @@ class WcsPublisher(QommonPublisher):
 
         wcs.workflows.load_extra()
 
-        if self.is_using_postgresql() and not skip_sql:
+        if self.has_postgresql_config() and not skip_sql:
             from . import sql
 
             self.user_class = sql.SqlUser