From 9f183d7e0a655ae7c24abd5442db3bb962d06c78 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 29 Apr 2022 19:24:07 +0200 Subject: [PATCH] clean: remove files and take partner's info on command line --- petale/api_views.py | 15 ------- petale/management/commands/clean.py | 62 +++++++++++++++++++++++------ petale/models.py | 15 ++++++- tests/conftest.py | 19 +++++---- tests/settings.py | 1 + tests/test_api.py | 4 +- tests/test_commands.py | 51 +++++++++++++++--------- 7 files changed, 108 insertions(+), 59 deletions(-) diff --git a/petale/api_views.py b/petale/api_views.py index 516f43b..2c6374c 100644 --- a/petale/api_views.py +++ b/petale/api_views.py @@ -55,21 +55,6 @@ logger = logging.getLogger(__name__) SENTINEL = object() -def check_unknown_cuts(uuids, creds=SENTINEL): - authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None) - if not authentic_url: - raise ValueError('PETALE_AUTHENTIC SETTINGS improperly defined') - - authentic_creds = creds if creds is not SENTINEL else getattr(settings, 'PETALE_AUTHENTIC_AUTH', None) - if not authentic_creds: - raise ValueError('missing credentials for authentic, configure PETALE_AUTHENTIC_AUTH') - url = urlparse.urljoin(authentic_url, 'api/users/synchronization/') - response = requests.post(url, json={"known_uuids": list(uuids)}, auth=authentic_creds, verify=False) - response.raise_for_status() - data = response.json() - return data.get("unknown_uuids") or [] - - def cut_exists(request, cut_uuid): if not getattr(settings, 'PETALE_CHECK_CUT_UUID', True): CUT.objects.get_or_create(uuid=cut_uuid) diff --git a/petale/management/commands/clean.py b/petale/management/commands/clean.py index 5d87690..4b1c9a2 100644 --- a/petale/management/commands/clean.py +++ b/petale/management/commands/clean.py @@ -14,12 +14,29 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import collections import itertools +import shutil +from urllib.parse import urljoin -from django.core.management.base import BaseCommand +import requests +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction -from petale import models -from petale.api_views import check_unknown_cuts +from petale.models import CUT, Partner, cut_path + + +def check_unknown_cuts(uuids, creds): + authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None) + if not authentic_url: + raise ValueError('PETALE_AUTHENTIC SETTINGS improperly defined') + + url = urljoin(authentic_url, 'api/users/synchronization/') + response = requests.post(url, json={"known_uuids": list(uuids)}, auth=creds, verify=False) + response.raise_for_status() + data = response.json() + return data.get("unknown_uuids") or [] def grouper(n, iterable, fillvalue=None): @@ -33,19 +50,38 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('--delete', action='store_true', help='Delete dead petals.') + parser.add_argument('partner_name') + parser.add_argument('partner_client_id') + parser.add_argument('partner_client_secret') - def handle(self, *args, delete=False, **options): + def handle(self, partner_name, partner_client_id, partner_client_secret, delete=False, **options): + try: + partner = Partner.objects.get(name=partner_name) + except Partner.DoesNotExist: + raise CommandError('partner %r does not exist' % partner_name) + + cuts = CUT.objects.filter(petal__partner=partner).distinct() zombie_uuids = set() - for cuts in grouper(500, models.CUT.objects.iterator()): - zombie_uuids.update(check_unknown_cuts({cut.uuid for cut in cuts if cut is not None})) - if options['verbosity'] > 1: - print('Found %d dead cuts on %d known cuts.' % (len(zombie_uuids), models.CUT.objects.count())) + for uuids in grouper(500, cuts.values_list('uuid', flat=True).iterator()): + # remove None + uuids = {uuid for uuid in uuids if uuid} + zombie_uuids.update(check_unknown_cuts(uuids, (partner_client_id, partner_client_secret))) + if options['verbosity'] > 1 or delete: + print('Found %d dead cuts on %d known cuts.' % (len(zombie_uuids), cuts.count())) if delete: - count, count_by_models = models.CUT.objects.filter(uuid__in=zombie_uuids).delete() - if count and options['verbosity'] > 0: + counts = collections.Counter() + try: + qs = CUT.objects.filter(uuid__in=zombie_uuids) + total = qs.count() + for i, cut in enumerate(qs): + print('Deleting %06d on %06d cuts.' % (i, total), end='\r') + with transaction.atomic(): + count, count_by_models = cut.delete() + shutil.rmtree(cut_path(partner, cut.uuid)) + counts += collections.Counter(count_by_models) + finally: + print() print( 'Deleted ', - ', '.join( - '%d %s' % (count, model.split('.')[-1]) for model, count in count_by_models.items() - ), + ', '.join('%d %s' % (count, model.split('.')[-1]) for model, count in counts.items()), ) diff --git a/petale/models.py b/petale/models.py index 67834d8..1628359 100644 --- a/petale/models.py +++ b/petale/models.py @@ -18,9 +18,10 @@ import hashlib from django.contrib.auth.models import User +from django.core.files.storage import default_storage from django.core.mail import send_mail from django.core.validators import RegexValidator -from django.db import models +from django.db import models, transaction from django.utils.translation import ugettext_lazy as _ from . import utils @@ -83,6 +84,12 @@ class Partner(models.Model): ordering = ['name'] +def cut_path(partner, uuid): + return default_storage.path( + 'data/{}/{}/{}/'.format(partner.name, hashlib.md5(uuid.encode('ascii')).hexdigest()[:3], uuid) + ) + + class CUT(models.Model): uuid = models.CharField(max_length=255, validators=[id_validator], unique=True) @@ -155,6 +162,12 @@ class Petal(models.Model): key=self.name, ) + @transaction.atomic(savepoint=False) + def delete(self, *args, **kwargs): + result = super().delete(*args, **kwargs) + self.data.delete(save=False) + return result + class Meta: unique_together = ('name', 'partner', 'cut') verbose_name = _('Petal') diff --git a/tests/conftest.py b/tests/conftest.py index ac0bcaf..bc0d69f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,3 @@ -import shutil -from tempfile import mkdtemp - import django_webtest import pytest from utils import ( @@ -14,17 +11,19 @@ from utils import ( ) +@pytest.fixture(autouse=True) +def media(settings, tmp_path): + media_path = tmp_path / 'media' + media_path.mkdir() + settings.MEDIA_ROOT = str(media_path) + return media_path + + @pytest.fixture -def app(request, settings): +def app(request, media): wtm = django_webtest.WebTestMixin() wtm._patch_settings() request.addfinalizer(wtm._unpatch_settings) - settings.MEDIA_ROOT = mkdtemp() - - def fin(): - shutil.rmtree(settings.MEDIA_ROOT) - - request.addfinalizer(fin) return django_webtest.DjangoTestApp() diff --git a/tests/settings.py b/tests/settings.py index d8a3cfb..ab9cb1f 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -14,3 +14,4 @@ if 'postgres' in DATABASES['default']['ENGINE']: PETALE_AUTHENTIC_URL = 'http://example.net/idp/' PETALE_AUTHENTIC_AUTH = ('foo', 'bar') +MEDIA_ROOT = '/does-not-exist/' diff --git a/tests/test_api.py b/tests/test_api.py index cf78f96..7e7975c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -15,7 +15,7 @@ pytestmark = pytest.mark.django_db def test_authentication_failure(app): - resp = app.get('/api/partner/12345abcde12345abcde12345abcde12/', status=401) + resp = app.get('/api/partner/12345abcde12345abcde12345abcde12/', status=403) json.loads(resp.text) @@ -363,7 +363,7 @@ def test_idp_based_partner_authentication(mocked_post, app, cut_kevin_uuid, acl) # failure response = {"result": 0, "errors": ["Invalid username/password."]} mocked_post.return_value = FakedResponse(content=json.dumps(response)) - app.put_json(url, params=payload, status=401, headers={'If-None-Match': '*'}) + app.put_json(url, params=payload, status=403, headers={'If-None-Match': '*'}) response = {"result": 1, "errors": []} # test with RP client_id = client_secret = 'a1b2' * 8 diff --git a/tests/test_commands.py b/tests/test_commands.py index b82c613..ce77f44 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -4,16 +4,10 @@ import pytest from django.core.files.base import ContentFile from django.core.management import call_command -from petale.api_views import check_unknown_cuts +from petale.management.commands.clean import check_unknown_cuts from petale.models import CUT, Partner, Petal -@pytest.fixture(autouse=True) -def settings(settings): - settings.PETALE_AUTHENTIC_URL = 'https://authentic.example.com/' - settings.PETALE_AUTHENTIC_AUTH = 'admin:admin' - - @pytest.fixture def requests_post(): post_return_value = mock.Mock() @@ -25,10 +19,10 @@ def requests_post(): def test_check_unknown_cuts(requests_post): requests_post.return_value.json.return_value = {'unknown_uuids': ['1']} - assert check_unknown_cuts(['1']) == ['1'] - assert requests_post.call_args[0] == ('https://authentic.example.com/api/users/synchronization/',) + assert check_unknown_cuts(['1'], ('admin', 'admin')) == ['1'] + assert requests_post.call_args[0] == ('http://example.net/idp/api/users/synchronization/',) assert requests_post.call_args[1] == { - 'auth': 'admin:admin', + 'auth': ('admin', 'admin'), 'json': {'known_uuids': ['1']}, 'verify': False, } @@ -39,31 +33,52 @@ class TestClean: uuids = {str(i) for i in range(10)} unknown_uuids = {str(i) for i in range(5)} + @pytest.fixture + def other_partner(self, db): + return Partner.objects.create(name='other') + @pytest.fixture def partner(self, db): return Partner.objects.create(name='partner') @pytest.fixture - def cuts(self, partner): + def cuts(self, partner, other_partner, media): + for uuid in ['a', 'b', 'c']: + cut = CUT.objects.create(uuid=uuid) + petal = Petal.objects.create(name='petal', partner=other_partner, cut=cut, size=0) + petal.data.save('petal.dat', ContentFile(b'1234')) for uuid in self.uuids: cut = CUT.objects.create(uuid=uuid) petal = Petal.objects.create(name='petal', partner=partner, cut=cut, size=0) petal.data.save('petal.dat', ContentFile(b'1234')) + assert len(list(media.rglob('petal'))) == 13 + assert len(list(media.glob('*/*/*/*'))) == 13 @pytest.fixture def requests_post(self, requests_post): requests_post.return_value.json.return_value = {'unknown_uuids': self.unknown_uuids} return requests_post - def test_dry(self, cuts, partner, requests_post): - call_command('clean') + def test_dry(self, cuts, partner, requests_post, media): + call_command('clean', 'partner', '1234', '5678') assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids - assert set(CUT.objects.values_list('uuid', flat=True)) == self.uuids + assert set(CUT.objects.filter(petal__partner=partner).values_list('uuid', flat=True)) == self.uuids + assert len(list(media.rglob('petal'))) == 13 + assert len(list(media.glob('*/*/*/*'))) == 13 - def test_delete(self, cuts, partner, requests_post): - call_command('clean', '--delete') + def test_delete(self, cuts, partner, requests_post, media): + call_command('clean', '--delete', 'partner', '1234', '5678') assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids - assert set(CUT.objects.values_list('uuid', flat=True)) == self.uuids - self.unknown_uuids + assert ( + set(CUT.objects.filter(petal__partner=partner).values_list('uuid', flat=True)) + == self.uuids - self.unknown_uuids + ) + assert len(list(media.rglob('petal'))) == 8 + assert len(list(media.glob('*/*/*/*'))) == 8 - call_command('clean', '--delete') + call_command('clean', '--delete', 'partner', '1234', '5678') assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids - self.unknown_uuids + assert len(list(media.rglob('*/partner/**/petal'))) == 5 + assert len(list(media.glob('*/partner/*/*'))) == 5 + assert len(list(media.rglob('*/*/**/petal'))) == 8 + assert len(list(media.glob('*/*/*/*'))) == 8