don't start a new cron if locked (#18519)

tests/test_publisher.py

@@ -8,8 +8,10 @@ import os
 import zipfile
 
 import mock
+import pytest
 
 from django.core.management import call_command
+from django.core.management.base import CommandError
 from quixote import cleanup
 from wcs.qommon.http_request import HTTPRequest
 
@@ -172,3 +174,29 @@ def test_cron_command():
     with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
         call_command('cron')
         assert cron_worker.call_count == 1
+
+    # simulate another locked cron
+    import tempfile
+    from qommon.vendor import locket
+    lockfile = os.path.join(tempfile.gettempdir(), 'wcs-cron-in-progress.lock')
+    with locket.lock_file(lockfile, timeout=0):
+        with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
+            call_command('cron')  # silent by default (verbosity=0)
+            assert cron_worker.call_count == 0
+            call_command('cron', verbosity=2)  # same if verbosity>0
+            assert cron_worker.call_count == 0
+    # verify that the lock is released
+    with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
+        call_command('cron')
+        assert cron_worker.call_count == 1
+
+    # simulate a cron crash
+    with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
+        cron_worker.side_effect = NotImplementedError
+        with pytest.raises(NotImplementedError):
+            call_command('cron')
+        assert cron_worker.call_count == 1
+    # verify that the lock is released
+    with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
+        call_command('cron')
+        assert cron_worker.call_count == 1

wcs/qommon/management/commands/cron.py

@@ -28,13 +28,27 @@ from qommon.cron import cron_worker
 class Command(BaseCommand):
     help = 'Execute cronjobs'
 
-    def handle(self, verbosity, **options):
-        with locket.lock_file(os.path.join(tempfile.gettempdir(), 'wcs-cron')):
-            now = time.localtime()
-            publisher_class = get_publisher_class()
-            publisher_class.register_cronjobs()
-            publisher = publisher_class.create_publisher()
-            app_dir = publisher.app_dir
-            for hostname in publisher.get_tenants():
-                publisher.app_dir = os.path.join(app_dir, hostname)
-                cron_worker(publisher, now)
+    def add_arguments(self, parser):
+        parser.set_defaults(verbosity=0)
+
+    def handle(self, verbosity, **kwargs):
+        lockfile = os.path.join(tempfile.gettempdir(), 'wcs-cron-in-progress.lock')
+        try:
+            with locket.lock_file(lockfile, timeout=0):
+                if verbosity > 2:
+                    print('cron start (lock %s)' % lockfile)
+                now = time.localtime()
+                publisher_class = get_publisher_class()
+                publisher_class.register_cronjobs()
+                publisher = publisher_class.create_publisher()
+                app_dir = publisher.app_dir
+                for hostname in publisher.get_tenants():
+                    if verbosity > 1:
+                        print('cron work on %s' % hostname)
+                    publisher.app_dir = os.path.join(app_dir, hostname)
+                    cron_worker(publisher, now)
+            if verbosity > 2:
+                print('cron end (release lock %s)' % lockfile)
+        except locket.LockError:
+            if verbosity > 0:
+                print('can not start cron job, locked by %s' % lockfile)