diff --git a/petale/api_views.py b/petale/api_views.py index 777ecc1..ed9e589 100644 --- a/petale/api_views.py +++ b/petale/api_views.py @@ -50,10 +50,27 @@ from .exceptions import ( from .models import CUT, AccessControlList, Partner, Petal from .utils import StreamingHash, logit +logger = logging.getLogger(__name__) + +SENTINEL = object() + + +def check_unknown_cuts(uuids, creds=SENTINEL): + authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None) + if not authentic_url: + raise ValueError('PETALE_AUTHENTIC SETTINGS improperly defined') + + authentic_creds = creds if creds is not SENTINEL else getattr(settings, 'PETALE_AUTHENTIC_CREDS', None) + if not authentic_creds: + raise ValueError('missing credentials for authentic, configure PETALE_AUTHENTIC_CREDS') + url = urlparse.urljoin(authentic_url, 'api/users/synchronization/') + response = requests.post(url, json={"known_uuids": list(uuids)}, auth=authentic_creds, verify=False) + response.raise_for_status() + data = response.json() + return data.get("unknown_uuids") or [] + def cut_exists(request, cut_uuid): - logger = logging.getLogger(__name__) - if not getattr(settings, 'PETALE_CHECK_CUT_UUID', True): CUT.objects.get_or_create(uuid=cut_uuid) return True diff --git a/petale/management/__init__.py b/petale/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/petale/management/commands/__init__.py b/petale/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/petale/management/commands/clean.py b/petale/management/commands/clean.py new file mode 100644 index 0000000..5d87690 --- /dev/null +++ b/petale/management/commands/clean.py @@ -0,0 +1,51 @@ +# Petale - Simple App as Key/Value Storage Interface +# Copyright (C) 2017 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import itertools + +from django.core.management.base import BaseCommand + +from petale import models +from petale.api_views import check_unknown_cuts + + +def grouper(n, iterable, fillvalue=None): + "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" + args = [iter(iterable)] * n + return itertools.zip_longest(fillvalue=fillvalue, *args) + + +class Command(BaseCommand): + help = 'Clean petale data' + + def add_arguments(self, parser): + parser.add_argument('--delete', action='store_true', help='Delete dead petals.') + + def handle(self, *args, delete=False, **options): + zombie_uuids = set() + for cuts in grouper(500, models.CUT.objects.iterator()): + zombie_uuids.update(check_unknown_cuts({cut.uuid for cut in cuts if cut is not None})) + if options['verbosity'] > 1: + print('Found %d dead cuts on %d known cuts.' % (len(zombie_uuids), models.CUT.objects.count())) + if delete: + count, count_by_models = models.CUT.objects.filter(uuid__in=zombie_uuids).delete() + if count and options['verbosity'] > 0: + print( + 'Deleted ', + ', '.join( + '%d %s' % (count, model.split('.')[-1]) for model, count in count_by_models.items() + ), + ) diff --git a/petale/migrations/0006_auto_20171017_1625.py b/petale/migrations/0006_auto_20171017_1625.py index 9182cb6..a8d3ed2 100644 --- a/petale/migrations/0006_auto_20171017_1625.py +++ b/petale/migrations/0006_auto_20171017_1625.py @@ -11,12 +11,14 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='partner', name='hard_global_max_size', - field=models.IntegerField(help_text='as kilobytes', verbose_name='Hard max size'), + field=models.IntegerField(default=0, help_text='as kilobytes', verbose_name='Hard max size'), ), migrations.AlterField( model_name='partner', name='hard_per_key_max_size', - field=models.IntegerField(help_text='as kilobytes', verbose_name='Hard max size per key'), + field=models.IntegerField( + default=0, help_text='as kilobytes', verbose_name='Hard max size per key' + ), ), migrations.AlterField( model_name='partner', @@ -26,11 +28,13 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='partner', name='soft_global_max_size', - field=models.IntegerField(help_text='as kilobytes', verbose_name='Soft max size'), + field=models.IntegerField(default=0, help_text='as kilobytes', verbose_name='Soft max size'), ), migrations.AlterField( model_name='partner', name='soft_per_key_max_size', - field=models.IntegerField(help_text='as kilobytes', verbose_name='Soft max size per key'), + field=models.IntegerField( + default=0, help_text='as kilobytes', verbose_name='Soft max size per key' + ), ), ] diff --git a/petale/models.py b/petale/models.py index 35d401e..67834d8 100644 --- a/petale/models.py +++ b/petale/models.py @@ -37,13 +37,17 @@ class Partner(models.Model): blank=True, help_text=_('List of admin emails separated by comma'), ) - hard_global_max_size = models.IntegerField(verbose_name=_('Hard max size'), help_text=_('as kilobytes')) - soft_global_max_size = models.IntegerField(verbose_name=_('Soft max size'), help_text=_('as kilobytes')) + hard_global_max_size = models.IntegerField( + verbose_name=_('Hard max size'), default=0, help_text=_('as kilobytes') + ) + soft_global_max_size = models.IntegerField( + verbose_name=_('Soft max size'), default=0, help_text=_('as kilobytes') + ) hard_per_key_max_size = models.IntegerField( - verbose_name=_('Hard max size per key'), help_text=_('as kilobytes') + verbose_name=_('Hard max size per key'), default=0, help_text=_('as kilobytes') ) soft_per_key_max_size = models.IntegerField( - verbose_name=_('Soft max size per key'), help_text=_('as kilobytes') + verbose_name=_('Soft max size per key'), default=0, help_text=_('as kilobytes') ) size = models.BigIntegerField(verbose_name=_('Size'), default=0, help_text=_('as bytes')) diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..65f8c97 --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,69 @@ +from unittest import mock + +import pytest +from django.core.files.base import ContentFile +from django.core.management import call_command + +from petale.api_views import check_unknown_cuts +from petale.models import CUT, Partner, Petal + + +@pytest.fixture(autouse=True) +def settings(settings): + settings.PETALE_AUTHENTIC_URL = 'https://authentic.example.com/' + settings.PETALE_AUTHENTIC_CREDS = 'admin:admin' + + +@pytest.fixture +def requests_post(): + post_return_value = mock.Mock() + post_return_value.json.return_value = None + with mock.patch('requests.post', return_value=post_return_value) as requests_post: + yield requests_post + + +def test_check_unknown_cuts(requests_post): + + requests_post.return_value.json.return_value = {'unknown_uuids': ['1']} + assert check_unknown_cuts(['1']) == ['1'] + assert requests_post.call_args[0] == ('https://authentic.example.com/api/users/synchronization/',) + assert requests_post.call_args[1] == { + 'auth': 'admin:admin', + 'json': {'known_uuids': ['1']}, + 'verify': False, + } + + +class TestClean: + + uuids = {str(i) for i in range(10)} + unknown_uuids = {str(i) for i in range(5)} + + @pytest.fixture + def partner(self, db): + return Partner.objects.create(name='partner') + + @pytest.fixture + def cuts(self, partner): + for uuid in self.uuids: + cut = CUT.objects.create(uuid=uuid) + petal = Petal.objects.create(name='petal', partner=partner, cut=cut, size=0) + petal.data.save('petal.dat', ContentFile(b'1234')) + + @pytest.fixture + def requests_post(self, requests_post): + requests_post.return_value.json.return_value = {'unknown_uuids': self.unknown_uuids} + return requests_post + + def test_dry(self, cuts, partner, requests_post): + call_command('clean') + assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids + assert set(CUT.objects.values_list('uuid', flat=True)) == self.uuids + + def test_delete(self, cuts, partner, requests_post): + call_command('clean', '--delete') + assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids + assert set(CUT.objects.values_list('uuid', flat=True)) == self.uuids - self.unknown_uuids + + call_command('clean', '--delete') + assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids - self.unknown_uuids