general: get carddef/formdef from publisher cache in more cases (#88983)

tests/test_carddef.py

@@ -624,6 +624,7 @@ def test_data_source_custom_view_digest(pub):
         'custom-view:view': '{{ form_var_foo }} Foo Bar',
     }
     carddef.store()
+    pub.reset_caches()
     # rebuild digests
     carddata.store()
     carddata2.store()
@@ -761,6 +762,7 @@ def test_get_data_source_custom_view_order_by(pub):
     ]
     carddef.digest_templates['custom-view:view'] = '{{ form_var_bar }}'
     carddef.store()
+    pub.reset_caches()
     for carddata in carddef.data_class().select():
         carddata.store()  # rebuild digests
     assert [i['text'] for i in CardDef.get_data_source_items('carddef:foo:view')] == [
@@ -1386,6 +1388,7 @@ def test_card_update_related(pub):
         ItemsField(id='1', label='Test', data_source={'type': 'carddef:foo'}),
     ]
     formdef.store()
+    pub.reset_caches()
 
     formdata = formdef.data_class()()
     formdata.data = {'1': ['1', '2']}
@@ -1419,6 +1422,7 @@ def test_card_update_related(pub):
         BlockField(id='2', label='Test2', block_slug=blockdef.slug),  # left empty
     ]
     formdef.store()
+    pub.reset_caches()
 
     formdata = formdef.data_class()()
     formdata.data = {

tests/test_formdata.py

@@ -5776,6 +5776,7 @@ def test_reverse_links(pub):
 
     # test reverse relation
     carddef1.store()  # build & store reverse_relations
+    pub.reset_caches()
     pub.substitutions.reset()
     pub.substitutions.feed(pub)
     pub.substitutions.feed(carddef1)
@@ -5790,6 +5791,7 @@ def test_reverse_links(pub):
     # test with natural id
     carddef1.id_template = 'X{{ form_var_name1 }}Y'
     carddef1.store()
+    pub.reset_caches()
     carddata1.store()
     assert carddata1.id_display == 'Xfoo1Y'
     carddata2.data['1'] = carddata1.get_natural_key()

tests/test_storage.py

@@ -31,6 +31,15 @@ class Foobar(StorableObject):
     unique_value = None
 
 
+class Foobar2(StorableObject):
+    _names = 'tests%s' % random.randint(0, 100000)
+    _indexes = ['unique_value']
+    _hashed_indexes = ['value']
+
+    value = None
+    unique_value = None
+
+
 def test_store():
     test = Foobar()
     test.value = 'value'
@@ -307,3 +316,30 @@ def test_umask():
     cache_umask()
     test.store()
     assert (os.stat(test.get_object_filename()).st_mode % 0o1000) == 0o664
+
+
+def test_publisher_cache():
+    pub.reset_caches()
+
+    Foobar.wipe()
+    Foobar2.wipe()
+
+    test = Foobar()
+    test.value = 'value'
+    test.unique_value = 'unique-value'
+    test.store()
+
+    test2 = Foobar2()
+    test2.value = 'value'
+    test2.unique_value = 'unique-value'
+    test2.store()
+
+    test = Foobar.cached_get('1')
+    assert test.value == 'value'
+    assert Foobar.cached_get('1') is test  # same object
+
+    assert Foobar.get_on_index('unique-value', 'unique_value') is not test
+    assert Foobar.get_on_index('unique-value', 'unique_value', use_cache=True) is test
+
+    assert Foobar2.cached_get('1') is not test
+    assert Foobar2.cached_get('1') is Foobar2.get_on_index('unique-value', 'unique_value', use_cache=True)

tests/utilities.py

@@ -146,6 +146,7 @@ def create_temporary_pub(pickle_mode=False, lazy_mode=False):
             sql.Audit.wipe()
             sql_mark_current_test()
         pub.write_cfg()
+        pub.reset_caches()
         return pub
 
     os.symlink(os.path.join(os.path.dirname(__file__), 'templates'), os.path.join(pub.app_dir, 'templates'))

wcs/carddata.py

@@ -156,7 +156,7 @@ class UpdateRelationsAfterJob(AfterJob):
         update_related_seen = get_publisher()._update_related_seen
 
         try:
-            carddef = CardDef.get(self.kwargs['carddef_id'])
+            carddef = CardDef.cached_get(self.kwargs['carddef_id'])
             carddata = carddef.data_class().get(self.kwargs['carddata_id'])
         except KeyError:
             # card got removed (probably the afterjob met some unexpected delay), ignore.
@@ -169,7 +169,7 @@ class UpdateRelationsAfterJob(AfterJob):
             obj_type, obj_slug = obj_ref.split(':')
             obj_class = klass.get(obj_type)
             try:
-                objdef = obj_class.get_by_slug(obj_slug)
+                objdef = obj_class.get_by_slug(obj_slug, use_cache=True)
             except KeyError:
                 continue
             criterias = []

wcs/carddef.py

@@ -204,7 +204,7 @@ class CardDef(FormDef):
         assert data_source_id.startswith('carddef:')
         parts = data_source_id.split(':')
         try:
-            carddef = cls.get_by_urlname(parts[1])
+            carddef = cls.get_by_urlname(parts[1], use_cache=True)
         except KeyError:
             return []
         criterias = [StrictNotEqual('status', 'draft'), Null('anonymised')]
@@ -299,7 +299,7 @@ class CardDef(FormDef):
         if len(parts) != 3:
             return []
         try:
-            carddef = cls.get_by_urlname(parts[1])
+            carddef = cls.get_by_urlname(parts[1], use_cache=True)
         except KeyError:
             return []
         custom_view = cls.get_data_source_custom_view(data_source_id, carddef=carddef)

wcs/formdata.py

@@ -1940,7 +1940,7 @@ class FormData(StorableObject):
                     elif obj_type == 'carddef':
                         obj_class = CardDef
                     try:
-                        _objectdef = obj_class.get_by_urlname(slug)
+                        _objectdef = obj_class.get_by_urlname(slug, use_cache=True)
                     except KeyError:
                         yield (
                             _('Linked object def by id %(object_id)s') % {'object_id': slug},

wcs/formdef.py

@@ -788,9 +788,13 @@ class FormDef(StorableObject):
         self.workflow_options.update(variables)
 
     @classmethod
-    def get_by_urlname(cls, url_name, ignore_migration=False, ignore_errors=False):
+    def get_by_urlname(cls, url_name, ignore_migration=False, ignore_errors=False, use_cache=False):
         return cls.get_on_index(
-            url_name, 'url_name', ignore_migration=ignore_migration, ignore_errors=ignore_errors
+            url_name,
+            'url_name',
+            ignore_migration=ignore_migration,
+            ignore_errors=ignore_errors,
+            use_cache=use_cache,
         )
 
     get_by_slug = get_by_urlname

wcs/qommon/storage.py

@@ -528,7 +528,7 @@ class StorableObject:
         return cls.sort_results(objects, order_by)
 
     @classmethod
-    def get_on_index(cls, id, index, ignore_errors=False, ignore_migration=False):
+    def get_on_index(cls, id, index, ignore_errors=False, ignore_migration=False, use_cache=False):
         if not cls._indexes:
             raise KeyError()
         objects_dir = cls.get_objects_dir()
@@ -536,6 +536,12 @@ class StorableObject:
         if not os.path.exists(index_dir):
             cls.rebuild_indexes()
         filename = os.path.join(index_dir, str(fix_key(id)))
+        if use_cache:
+            try:
+                object_id = os.readlink(filename).split('/')[-1]
+            except FileNotFoundError:
+                raise KeyError(id)
+            return cls.cached_get(object_id, ignore_errors=ignore_errors, ignore_migration=ignore_migration)
         return cls.get_filename(filename, ignore_errors=ignore_errors, ignore_migration=ignore_migration)
 
     @classmethod

wcs/variables.py

@@ -2041,7 +2041,7 @@ class CardsSource:
 
     def __getattr__(self, attr):
         try:
-            return LazyFormDef(CardDef.get_by_urlname(attr))
+            return LazyFormDef(CardDef.get_by_urlname(attr, use_cache=True))
         except KeyError:
             raise CardDefDoesNotExist(attr)
 
@@ -2056,6 +2056,6 @@ class FormsSource:
 
     def __getattr__(self, attr):
         try:
-            return LazyFormDef(FormDef.get_by_urlname(attr))
+            return LazyFormDef(FormDef.get_by_urlname(attr, use_cache=True))
         except KeyError:
             raise FormDefDoesNotExist(attr)

wcs/wf/create_formdata.py

@@ -214,9 +214,7 @@ class LinkedFormdataEvolutionPart(EvolutionPart):
 
     @property
     def formdef(self):
-        if not hasattr(self, '_formdef'):
-            self._formdef = self.formdef_class.get(self.formdef_id, ignore_errors=True)
-        return self._formdef
+        return self.formdef_class.cached_get(self.formdef_id, ignore_errors=True, ignore_migration=True)
 
     @property
     def formdata(self):

wcs/workflow_traces.py

@@ -104,13 +104,11 @@ class WorkflowTrace(sql.WorkflowTrace):
         from wcs.carddef import CardDef
         from wcs.formdef import FormDef
 
-        if not hasattr(self, '_formdef'):
-            formdef_class = FormDef
-            if 'carddata' in self.event:
-                formdef_class = CardDef
+        formdef_class = FormDef
+        if 'carddata' in self.event:
+            formdef_class = CardDef
 
-            self._formdef = formdef_class.get(self.event_args.get('external_formdef_id'), ignore_errors=True)
-        return self._formdef
+        return formdef_class.cached_get(self.event_args.get('external_formdef_id'), ignore_errors=True)
 
     @property
     def formdata(self):