general: move cron jobs in app configs (#28000)

combo/apps/dashboard/__init__.py

@@ -12,6 +12,7 @@
 # GNU Affero General Public License for more details.
 
 import django.apps
+from django.utils.timezone import now, timedelta
 from django.utils.translation import ugettext_lazy as _
 
 class AppConfig(django.apps.AppConfig):
@@ -22,4 +23,13 @@ class AppConfig(django.apps.AppConfig):
         from . import urls
         return urls.urlpatterns
 
+    def hourly(self):
+        self.clean_autotiles()
+
+    def clean_autotiles(self):
+        from combo.data.models import ConfigJsonCell
+        ConfigJsonCell.objects.filter(placeholder='_auto_tile',
+                last_update_timestamp__lte=now() - timedelta(days=2)).delete()
+
+
 default_app_config = 'combo.apps.dashboard.AppConfig'

combo/apps/dashboard/management/commands/clean_autotiles.py

@@ -1,28 +0,0 @@
-# combo - content management system
-# Copyright (C) 2014-2018  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now, timedelta
-
-from combo.data.models import ConfigJsonCell
-
-
-class Command(BaseCommand):
-    help = 'Delete automatically created tile cells that are more than 2 days old'
-
-    def handle(self, *args, **options):
-        ConfigJsonCell.objects.filter(placeholder='_auto_tile',
-                last_update_timestamp__lte=now() - timedelta(days=2)).delete()

combo/apps/lingo/__init__.py

@@ -14,8 +14,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
+import logging
+
 import django.apps
 from django.core.urlresolvers import reverse
+from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
 
@@ -31,4 +35,35 @@ class AppConfig(django.apps.AppConfig):
         return [{'href': reverse('lingo-manager-homepage'),
                 'text': _('Online Payment')}]
 
+    def hourly(self):
+        self.update_transactions()
+
+    def update_transactions(self):
+        from .models import Transaction, EXPIRED
+        logger = logging.getLogger(__name__)
+        now = timezone.now()
+        for transaction in Transaction.objects.filter(
+                start_date__lt=now-datetime.timedelta(hours=1),
+                end_date__isnull=True):
+            logger.info('transaction %r is expired', transaction.order_id)
+            transaction.status = EXPIRED
+            transaction.save()
+
+        for transaction in Transaction.objects.filter(to_be_paid_remote_items__isnull=False):
+            transaction.retry_notify_remote_items_of_payments()
+
+    def notify_payments(self):
+        from combo.apps.lingo.models import BasketItem
+        logger = logging.getLogger(__name__)
+        now = timezone.now()
+        for item in BasketItem.objects.filter(
+                notification_date__isnull=True,
+                cancellation_date__isnull=True,
+                payment_date__lt=now-datetime.timedelta(minutes=5),
+                payment_date__gt=now-datetime.timedelta(minutes=300)):
+            try:
+                item.notify_payment()
+            except:
+                logger.exception('error in async notification for basket item %s', item.id)
+
 default_app_config = 'combo.apps.lingo.AppConfig'

combo/apps/lingo/management/commands/notify_payments.py

@@ -1,41 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# lingo - basket and payment system
-# Copyright (C) 2017  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import datetime
-import logging
-
-from django.core.management.base import BaseCommand
-from django.utils import timezone
-
-from combo.apps.lingo.models import BasketItem
-
-
-class Command(BaseCommand):
-
-    def handle(self, *args, **kwargs):
-        logger = logging.getLogger(__name__)
-        now = timezone.now()
-        for item in BasketItem.objects.filter(
-                notification_date__isnull=True,
-                cancellation_date__isnull=True,
-                payment_date__lt=now-datetime.timedelta(minutes=5),
-                payment_date__gt=now-datetime.timedelta(minutes=300)):
-            try:
-                item.notify_payment()
-            except:
-                logger.exception('error in async notification for basket item %s', item.id)

combo/apps/lingo/management/commands/update_transactions.py

@@ -1,22 +0,0 @@
-import logging
-from datetime import timedelta
-
-from django.utils import timezone
-from django.core.management.base import BaseCommand
-
-from combo.apps.lingo.models import Transaction, EXPIRED
-
-
-class Command(BaseCommand):
-
-    def handle(self, *args, **kwargs):
-        logger = logging.getLogger(__name__)
-        now = timezone.now()
-        for transaction in Transaction.objects.filter(start_date__lt=now-timedelta(hours=1),
-                       end_date__isnull=True):
-            logger.info('transaction %r is expired', transaction.order_id)
-            transaction.status = EXPIRED
-            transaction.save()
-
-        for transaction in Transaction.objects.filter(to_be_paid_remote_items__isnull=False):
-            transaction.retry_notify_remote_items_of_payments()

combo/apps/momo/__init__.py

@@ -17,6 +17,10 @@
 import django.apps
 from django.conf import settings
 from django.core.urlresolvers import reverse
+from django.db import connection
+from django.test.client import RequestFactory
+from django.utils import translation
+from django.utils.six.moves.urllib.parse import urlparse
 from django.utils.translation import ugettext_lazy as _
 
 class AppConfig(django.apps.AppConfig):
@@ -34,4 +38,27 @@ class AppConfig(django.apps.AppConfig):
         return [{'href': reverse('momo-manager-homepage'),
                 'text': _('Mobile Application')}]
 
+    def hourly(self):
+        from .utils import GenerationInfo
+        try:
+            self.update_momo_manifest()
+        except GenerationInfo:
+            pass
+
+    def update_momo_manifest(self):
+        from .utils import generate_manifest
+        tenant = connection.get_tenant()
+        parsed_base_url = urlparse(tenant.get_base_url())
+        if ':' in parsed_base_url.netloc:
+            server_name, server_port = parsed_base_url.netloc.split(':')
+        else:
+            server_name = parsed_base_url.netloc
+            server_port = '80' if parsed_base_url.scheme == 'http' else '443'
+        request = RequestFactory().get('/', SERVER_NAME=server_name,
+                SERVER_PORT=server_port)
+        request._get_scheme = lambda: parsed_base_url.scheme
+
+        with translation.override(settings.LANGUAGE_CODE):
+            generate_manifest(request)
+
 default_app_config = 'combo.apps.momo.AppConfig'

combo/apps/momo/management/commands/update_momo_manifest.py

@@ -1,49 +0,0 @@
-# combo - content management system
-# Copyright (C) 2016  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-from django.db import connection
-from django.test.client import RequestFactory
-from django.utils import translation
-from django.utils.six.moves.urllib.parse import urlparse
-
-from combo.apps.momo.utils import generate_manifest, GenerationError, GenerationInfo
-
-class Command(BaseCommand):
-    def handle(self, *args, **kwargs):
-        if not getattr(settings, 'ENABLE_MOMO', False):
-            return
-        tenant = connection.get_tenant()
-        parsed_base_url = urlparse(tenant.get_base_url())
-        if ':' in parsed_base_url.netloc:
-            server_name, server_port = parsed_base_url.netloc.split(':')
-        else:
-            server_name = parsed_base_url.netloc
-            server_port = '80' if parsed_base_url.scheme == 'http' else '443'
-        request = RequestFactory().get('/', SERVER_NAME=server_name,
-                SERVER_PORT=server_port)
-        request._get_scheme = lambda: parsed_base_url.scheme
-
-        translation.activate(settings.LANGUAGE_CODE)
-        try:
-            generate_manifest(request)
-        except GenerationError as e:
-            raise CommandError(e.message)
-        except GenerationInfo as e:
-            if int(kwargs.get('verbosity')) > 0:
-                print(e.message)
-        translation.deactivate()

combo/data/management/commands/cron.py

@@ -0,0 +1,49 @@
+# combo - content management system
+# Copyright (C) 2014-2018  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import traceback
+
+from django.apps import apps
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+
+class Command(BaseCommand):
+    help = 'Execute scheduled commands'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--application', dest='application', metavar='APPLICATION', type=str,
+                help='limit updates to given application')
+
+    def handle(self, **options):
+        errors = []
+        for appconfig in apps.get_app_configs():
+            if not hasattr(appconfig, 'hourly'):
+                continue
+            if hasattr(appconfig, 'is_enabled') and not appconfig.is_enabled():
+                continue
+            try:
+                appconfig.hourly()
+            except Exception as e:
+                errors.append({'application': appconfig.name, 'exception': e, 'traceback': traceback.format_exc()})
+        if errors:
+            for error in errors:
+                if options['verbosity'] >= 1:
+                    print '%s: error: %s' % (error['application'], error['exception'])
+                if options['verbosity'] >= 2:
+                    print error['traceback']
+                    print
+            raise CommandError('error running jobs')

debian/combo.cron.hourly

@@ -1,8 +1,5 @@
 #!/bin/sh
 
+/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command cron --all-tenants
 /sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command clearsessions --all-tenants
-/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command update_transactions --all-tenants
-/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command update_momo_manifest --all-tenants -v0
 /sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command update_index --remove --all-tenants -v0
-/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command notify_payments --all-tenants -v0
-/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command clean_autotiles --all-tenants -v0

tests/test_dashboard.py

@@ -4,6 +4,7 @@ import os
 import pytest
 from webtest import TestApp
 
+from django.apps import apps
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.management import call_command
@@ -208,4 +209,5 @@ def test_auto_tile(app, site):
             assert resp.text.strip() == '/var1=one/var2=/'
 
 def test_clean_autotiles(app, site):
-    call_command('clean_autotiles')
+    appconfig = apps.get_app_config('dashboard')
+    appconfig.clean_autotiles()

tests/test_lingo_payment.py

@@ -7,6 +7,7 @@ from decimal import Decimal
 import json
 import mock
 
+from django.apps import apps
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 from django.core.wsgi import get_wsgi_application
@@ -19,8 +20,6 @@ from webtest import TestApp
 from combo.data.models import Page
 from combo.apps.lingo.models import (Regie, BasketItem, Transaction,
         TransactionOperation, RemoteItem, EXPIRED, LingoBasketCell)
-from combo.apps.lingo.management.commands.update_transactions import Command as UpdateTransactionsCommand
-from combo.apps.lingo.management.commands.notify_payments import Command as NotifyPaymentsCommand
 from combo.utils import sign_url
 
 from .test_manager import login
@@ -674,8 +673,8 @@ def test_transaction_expiration():
     t2 = Transaction(status=0)
     t2.save()
 
-    cmd = UpdateTransactionsCommand()
-    cmd.handle()
+    appconfig = apps.get_app_config('lingo')
+    appconfig.update_transactions()
 
     assert Transaction.objects.get(id=t1.id).status == EXPIRED
     assert Transaction.objects.get(id=t2.id).status == 0
@@ -861,7 +860,8 @@ def test_payment_callback_error(app, basket_page, regie, user):
     assert not BasketItem.objects.get(id=item.id).notification_date
 
     # too soon
-    NotifyPaymentsCommand().handle()
+    appconfig = apps.get_app_config('lingo')
+    appconfig.notify_payments()
     assert BasketItem.objects.get(id=item.id).payment_date
     assert not BasketItem.objects.get(id=item.id).notification_date
 
@@ -874,7 +874,7 @@ def test_payment_callback_error(app, basket_page, regie, user):
         mock_response = mock.Mock()
         mock_response.status_code = 200
         request.return_value = mock_response
-        NotifyPaymentsCommand().handle()
+        appconfig.notify_payments()
         url = request.call_args[0][1]
         assert url.startswith('http://example.org/testitem/jump/trigger/paid')
     assert BasketItem.objects.get(id=item.id).payment_date

tests/test_lingo_remote_regie.py

@@ -6,6 +6,7 @@ import mock
 from decimal import Decimal
 from requests.exceptions import ConnectionError
 
+from django.apps import apps
 from django.test.client import RequestFactory
 from django.test import override_settings
 from django.core.urlresolvers import reverse
@@ -348,13 +349,14 @@ def test_remote_item_payment_failure(mock_post, mock_get, mock_pay_invoice, app,
     assert resp.status_code == 200
 
     mock_get.side_effect = None
-    call_command('update_transactions')
+    appconfig = apps.get_app_config('lingo')
+    appconfig.update_transactions()
 
     assert Transaction.objects.count() == 1
     assert BasketItem.objects.count() == 1
     assert Transaction.objects.all()[0].to_be_paid_remote_items is None
 
-    call_command('update_transactions')
+    appconfig.update_transactions()
 
 
 @mock.patch('combo.apps.lingo.models.Regie.pay_invoice')

tests/test_momo.py

@@ -8,7 +8,6 @@ from webtest import TestApp
 from django.conf import settings
 
 from combo.data.models import Page, CellBase, TextCell, LinkCell, FeedCell
-from combo.apps.momo.management.commands.update_momo_manifest import Command as UpdateCommand
 
 from .test_manager import login, admin_user