From ac33f94f81db6faf10a447dc4ce4a050645242a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Wed, 14 Nov 2018 17:37:42 +0100 Subject: [PATCH] general: move cron jobs in app configs (#28000) --- combo/apps/dashboard/__init__.py | 10 ++++ combo/apps/dashboard/management/__init__.py | 0 .../dashboard/management/commands/__init__.py | 0 .../management/commands/clean_autotiles.py | 28 ----------- combo/apps/lingo/__init__.py | 35 +++++++++++++ .../management/commands/notify_payments.py | 41 ---------------- .../commands/update_transactions.py | 22 --------- combo/apps/momo/__init__.py | 27 ++++++++++ combo/apps/momo/management/__init__.py | 0 .../apps/momo/management/commands/__init__.py | 0 .../commands/update_momo_manifest.py | 49 ------------------- combo/data/management/commands/cron.py | 49 +++++++++++++++++++ debian/combo.cron.hourly | 5 +- tests/test_dashboard.py | 4 +- tests/test_lingo_payment.py | 12 ++--- tests/test_lingo_remote_regie.py | 6 ++- tests/test_momo.py | 1 - 17 files changed, 135 insertions(+), 154 deletions(-) delete mode 100644 combo/apps/dashboard/management/__init__.py delete mode 100644 combo/apps/dashboard/management/commands/__init__.py delete mode 100644 combo/apps/dashboard/management/commands/clean_autotiles.py delete mode 100644 combo/apps/lingo/management/commands/notify_payments.py delete mode 100644 combo/apps/lingo/management/commands/update_transactions.py delete mode 100644 combo/apps/momo/management/__init__.py delete mode 100644 combo/apps/momo/management/commands/__init__.py delete mode 100644 combo/apps/momo/management/commands/update_momo_manifest.py create mode 100644 combo/data/management/commands/cron.py diff --git a/combo/apps/dashboard/__init__.py b/combo/apps/dashboard/__init__.py index b2b584e4..034752c1 100644 --- a/combo/apps/dashboard/__init__.py +++ b/combo/apps/dashboard/__init__.py @@ -12,6 +12,7 @@ # GNU Affero General Public License for more details. import django.apps +from django.utils.timezone import now, timedelta from django.utils.translation import ugettext_lazy as _ class AppConfig(django.apps.AppConfig): @@ -22,4 +23,13 @@ class AppConfig(django.apps.AppConfig): from . import urls return urls.urlpatterns + def hourly(self): + self.clean_autotiles() + + def clean_autotiles(self): + from combo.data.models import ConfigJsonCell + ConfigJsonCell.objects.filter(placeholder='_auto_tile', + last_update_timestamp__lte=now() - timedelta(days=2)).delete() + + default_app_config = 'combo.apps.dashboard.AppConfig' diff --git a/combo/apps/dashboard/management/__init__.py b/combo/apps/dashboard/management/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/combo/apps/dashboard/management/commands/__init__.py b/combo/apps/dashboard/management/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/combo/apps/dashboard/management/commands/clean_autotiles.py b/combo/apps/dashboard/management/commands/clean_autotiles.py deleted file mode 100644 index 8c7c981b..00000000 --- a/combo/apps/dashboard/management/commands/clean_autotiles.py +++ /dev/null @@ -1,28 +0,0 @@ -# combo - content management system -# Copyright (C) 2014-2018 Entr'ouvert -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from django.core.management.base import BaseCommand -from django.utils.timezone import now, timedelta - -from combo.data.models import ConfigJsonCell - - -class Command(BaseCommand): - help = 'Delete automatically created tile cells that are more than 2 days old' - - def handle(self, *args, **options): - ConfigJsonCell.objects.filter(placeholder='_auto_tile', - last_update_timestamp__lte=now() - timedelta(days=2)).delete() diff --git a/combo/apps/lingo/__init__.py b/combo/apps/lingo/__init__.py index 0ef41e25..97c52872 100644 --- a/combo/apps/lingo/__init__.py +++ b/combo/apps/lingo/__init__.py @@ -14,8 +14,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import datetime +import logging + import django.apps from django.core.urlresolvers import reverse +from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -31,4 +35,35 @@ class AppConfig(django.apps.AppConfig): return [{'href': reverse('lingo-manager-homepage'), 'text': _('Online Payment')}] + def hourly(self): + self.update_transactions() + + def update_transactions(self): + from .models import Transaction, EXPIRED + logger = logging.getLogger(__name__) + now = timezone.now() + for transaction in Transaction.objects.filter( + start_date__lt=now-datetime.timedelta(hours=1), + end_date__isnull=True): + logger.info('transaction %r is expired', transaction.order_id) + transaction.status = EXPIRED + transaction.save() + + for transaction in Transaction.objects.filter(to_be_paid_remote_items__isnull=False): + transaction.retry_notify_remote_items_of_payments() + + def notify_payments(self): + from combo.apps.lingo.models import BasketItem + logger = logging.getLogger(__name__) + now = timezone.now() + for item in BasketItem.objects.filter( + notification_date__isnull=True, + cancellation_date__isnull=True, + payment_date__lt=now-datetime.timedelta(minutes=5), + payment_date__gt=now-datetime.timedelta(minutes=300)): + try: + item.notify_payment() + except: + logger.exception('error in async notification for basket item %s', item.id) + default_app_config = 'combo.apps.lingo.AppConfig' diff --git a/combo/apps/lingo/management/commands/notify_payments.py b/combo/apps/lingo/management/commands/notify_payments.py deleted file mode 100644 index 9749fd70..00000000 --- a/combo/apps/lingo/management/commands/notify_payments.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -# -# lingo - basket and payment system -# Copyright (C) 2017 Entr'ouvert -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import datetime -import logging - -from django.core.management.base import BaseCommand -from django.utils import timezone - -from combo.apps.lingo.models import BasketItem - - -class Command(BaseCommand): - - def handle(self, *args, **kwargs): - logger = logging.getLogger(__name__) - now = timezone.now() - for item in BasketItem.objects.filter( - notification_date__isnull=True, - cancellation_date__isnull=True, - payment_date__lt=now-datetime.timedelta(minutes=5), - payment_date__gt=now-datetime.timedelta(minutes=300)): - try: - item.notify_payment() - except: - logger.exception('error in async notification for basket item %s', item.id) diff --git a/combo/apps/lingo/management/commands/update_transactions.py b/combo/apps/lingo/management/commands/update_transactions.py deleted file mode 100644 index 08a43f80..00000000 --- a/combo/apps/lingo/management/commands/update_transactions.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging -from datetime import timedelta - -from django.utils import timezone -from django.core.management.base import BaseCommand - -from combo.apps.lingo.models import Transaction, EXPIRED - - -class Command(BaseCommand): - - def handle(self, *args, **kwargs): - logger = logging.getLogger(__name__) - now = timezone.now() - for transaction in Transaction.objects.filter(start_date__lt=now-timedelta(hours=1), - end_date__isnull=True): - logger.info('transaction %r is expired', transaction.order_id) - transaction.status = EXPIRED - transaction.save() - - for transaction in Transaction.objects.filter(to_be_paid_remote_items__isnull=False): - transaction.retry_notify_remote_items_of_payments() diff --git a/combo/apps/momo/__init__.py b/combo/apps/momo/__init__.py index 37b1252a..94b5c6a6 100644 --- a/combo/apps/momo/__init__.py +++ b/combo/apps/momo/__init__.py @@ -17,6 +17,10 @@ import django.apps from django.conf import settings from django.core.urlresolvers import reverse +from django.db import connection +from django.test.client import RequestFactory +from django.utils import translation +from django.utils.six.moves.urllib.parse import urlparse from django.utils.translation import ugettext_lazy as _ class AppConfig(django.apps.AppConfig): @@ -34,4 +38,27 @@ class AppConfig(django.apps.AppConfig): return [{'href': reverse('momo-manager-homepage'), 'text': _('Mobile Application')}] + def hourly(self): + from .utils import GenerationInfo + try: + self.update_momo_manifest() + except GenerationInfo: + pass + + def update_momo_manifest(self): + from .utils import generate_manifest + tenant = connection.get_tenant() + parsed_base_url = urlparse(tenant.get_base_url()) + if ':' in parsed_base_url.netloc: + server_name, server_port = parsed_base_url.netloc.split(':') + else: + server_name = parsed_base_url.netloc + server_port = '80' if parsed_base_url.scheme == 'http' else '443' + request = RequestFactory().get('/', SERVER_NAME=server_name, + SERVER_PORT=server_port) + request._get_scheme = lambda: parsed_base_url.scheme + + with translation.override(settings.LANGUAGE_CODE): + generate_manifest(request) + default_app_config = 'combo.apps.momo.AppConfig' diff --git a/combo/apps/momo/management/__init__.py b/combo/apps/momo/management/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/combo/apps/momo/management/commands/__init__.py b/combo/apps/momo/management/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/combo/apps/momo/management/commands/update_momo_manifest.py b/combo/apps/momo/management/commands/update_momo_manifest.py deleted file mode 100644 index 68fddfc2..00000000 --- a/combo/apps/momo/management/commands/update_momo_manifest.py +++ /dev/null @@ -1,49 +0,0 @@ -# combo - content management system -# Copyright (C) 2016 Entr'ouvert -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.db import connection -from django.test.client import RequestFactory -from django.utils import translation -from django.utils.six.moves.urllib.parse import urlparse - -from combo.apps.momo.utils import generate_manifest, GenerationError, GenerationInfo - -class Command(BaseCommand): - def handle(self, *args, **kwargs): - if not getattr(settings, 'ENABLE_MOMO', False): - return - tenant = connection.get_tenant() - parsed_base_url = urlparse(tenant.get_base_url()) - if ':' in parsed_base_url.netloc: - server_name, server_port = parsed_base_url.netloc.split(':') - else: - server_name = parsed_base_url.netloc - server_port = '80' if parsed_base_url.scheme == 'http' else '443' - request = RequestFactory().get('/', SERVER_NAME=server_name, - SERVER_PORT=server_port) - request._get_scheme = lambda: parsed_base_url.scheme - - translation.activate(settings.LANGUAGE_CODE) - try: - generate_manifest(request) - except GenerationError as e: - raise CommandError(e.message) - except GenerationInfo as e: - if int(kwargs.get('verbosity')) > 0: - print(e.message) - translation.deactivate() diff --git a/combo/data/management/commands/cron.py b/combo/data/management/commands/cron.py new file mode 100644 index 00000000..61977f1c --- /dev/null +++ b/combo/data/management/commands/cron.py @@ -0,0 +1,49 @@ +# combo - content management system +# Copyright (C) 2014-2018 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import traceback + +from django.apps import apps +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + + +class Command(BaseCommand): + help = 'Execute scheduled commands' + + def add_arguments(self, parser): + parser.add_argument('--application', dest='application', metavar='APPLICATION', type=str, + help='limit updates to given application') + + def handle(self, **options): + errors = [] + for appconfig in apps.get_app_configs(): + if not hasattr(appconfig, 'hourly'): + continue + if hasattr(appconfig, 'is_enabled') and not appconfig.is_enabled(): + continue + try: + appconfig.hourly() + except Exception as e: + errors.append({'application': appconfig.name, 'exception': e, 'traceback': traceback.format_exc()}) + if errors: + for error in errors: + if options['verbosity'] >= 1: + print '%s: error: %s' % (error['application'], error['exception']) + if options['verbosity'] >= 2: + print error['traceback'] + print + raise CommandError('error running jobs') diff --git a/debian/combo.cron.hourly b/debian/combo.cron.hourly index c850f665..0ca14028 100644 --- a/debian/combo.cron.hourly +++ b/debian/combo.cron.hourly @@ -1,8 +1,5 @@ #!/bin/sh +/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command cron --all-tenants /sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command clearsessions --all-tenants -/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command update_transactions --all-tenants -/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command update_momo_manifest --all-tenants -v0 /sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command update_index --remove --all-tenants -v0 -/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command notify_payments --all-tenants -v0 -/sbin/runuser -u combo /usr/bin/combo-manage -- tenant_command clean_autotiles --all-tenants -v0 diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 25672826..94d2c593 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -4,6 +4,7 @@ import os import pytest from webtest import TestApp +from django.apps import apps from django.conf import settings from django.contrib.auth.models import User from django.core.management import call_command @@ -208,4 +209,5 @@ def test_auto_tile(app, site): assert resp.text.strip() == '/var1=one/var2=/' def test_clean_autotiles(app, site): - call_command('clean_autotiles') + appconfig = apps.get_app_config('dashboard') + appconfig.clean_autotiles() diff --git a/tests/test_lingo_payment.py b/tests/test_lingo_payment.py index 8ee1c7ec..ac1d4f57 100644 --- a/tests/test_lingo_payment.py +++ b/tests/test_lingo_payment.py @@ -7,6 +7,7 @@ from decimal import Decimal import json import mock +from django.apps import apps from django.contrib.auth.models import User from django.core.urlresolvers import reverse from django.core.wsgi import get_wsgi_application @@ -19,8 +20,6 @@ from webtest import TestApp from combo.data.models import Page from combo.apps.lingo.models import (Regie, BasketItem, Transaction, TransactionOperation, RemoteItem, EXPIRED, LingoBasketCell) -from combo.apps.lingo.management.commands.update_transactions import Command as UpdateTransactionsCommand -from combo.apps.lingo.management.commands.notify_payments import Command as NotifyPaymentsCommand from combo.utils import sign_url from .test_manager import login @@ -674,8 +673,8 @@ def test_transaction_expiration(): t2 = Transaction(status=0) t2.save() - cmd = UpdateTransactionsCommand() - cmd.handle() + appconfig = apps.get_app_config('lingo') + appconfig.update_transactions() assert Transaction.objects.get(id=t1.id).status == EXPIRED assert Transaction.objects.get(id=t2.id).status == 0 @@ -861,7 +860,8 @@ def test_payment_callback_error(app, basket_page, regie, user): assert not BasketItem.objects.get(id=item.id).notification_date # too soon - NotifyPaymentsCommand().handle() + appconfig = apps.get_app_config('lingo') + appconfig.notify_payments() assert BasketItem.objects.get(id=item.id).payment_date assert not BasketItem.objects.get(id=item.id).notification_date @@ -874,7 +874,7 @@ def test_payment_callback_error(app, basket_page, regie, user): mock_response = mock.Mock() mock_response.status_code = 200 request.return_value = mock_response - NotifyPaymentsCommand().handle() + appconfig.notify_payments() url = request.call_args[0][1] assert url.startswith('http://example.org/testitem/jump/trigger/paid') assert BasketItem.objects.get(id=item.id).payment_date diff --git a/tests/test_lingo_remote_regie.py b/tests/test_lingo_remote_regie.py index 42767923..67b8480b 100644 --- a/tests/test_lingo_remote_regie.py +++ b/tests/test_lingo_remote_regie.py @@ -6,6 +6,7 @@ import mock from decimal import Decimal from requests.exceptions import ConnectionError +from django.apps import apps from django.test.client import RequestFactory from django.test import override_settings from django.core.urlresolvers import reverse @@ -348,13 +349,14 @@ def test_remote_item_payment_failure(mock_post, mock_get, mock_pay_invoice, app, assert resp.status_code == 200 mock_get.side_effect = None - call_command('update_transactions') + appconfig = apps.get_app_config('lingo') + appconfig.update_transactions() assert Transaction.objects.count() == 1 assert BasketItem.objects.count() == 1 assert Transaction.objects.all()[0].to_be_paid_remote_items is None - call_command('update_transactions') + appconfig.update_transactions() @mock.patch('combo.apps.lingo.models.Regie.pay_invoice') diff --git a/tests/test_momo.py b/tests/test_momo.py index 9ff98eec..0a811ecc 100644 --- a/tests/test_momo.py +++ b/tests/test_momo.py @@ -8,7 +8,6 @@ from webtest import TestApp from django.conf import settings from combo.data.models import Page, CellBase, TextCell, LinkCell, FeedCell -from combo.apps.momo.management.commands.update_momo_manifest import Command as UpdateCommand from .test_manager import login, admin_user