misc: add clean command (#60348)

Delete CUT and Petal models for deleted GLC accounts.

petale/api_views.py

@@ -50,10 +50,27 @@ from .exceptions import (
 from .models import CUT, AccessControlList, Partner, Petal
 from .utils import StreamingHash, logit
 
+logger = logging.getLogger(__name__)
+
+SENTINEL = object()
+
+
+def check_unknown_cuts(uuids, creds=SENTINEL):
+    authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None)
+    if not authentic_url:
+        raise ValueError('PETALE_AUTHENTIC SETTINGS improperly defined')
+
+    authentic_creds = creds if creds is not SENTINEL else getattr(settings, 'PETALE_AUTHENTIC_CREDS', None)
+    if not authentic_creds:
+        raise ValueError('missing credentials for authentic, configure PETALE_AUTHENTIC_CREDS')
+    url = urlparse.urljoin(authentic_url, 'api/users/synchronization/')
+    response = requests.post(url, json={"known_uuids": list(uuids)}, auth=authentic_creds, verify=False)
+    response.raise_for_status()
+    data = response.json()
+    return data.get("unknown_uuids") or []
+
 
 def cut_exists(request, cut_uuid):
-    logger = logging.getLogger(__name__)
-
     if not getattr(settings, 'PETALE_CHECK_CUT_UUID', True):
         CUT.objects.get_or_create(uuid=cut_uuid)
         return True

petale/management/commands/clean.py

@@ -0,0 +1,51 @@
+# Petale - Simple App as Key/Value Storage Interface
+# Copyright (C) 2017  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import itertools
+
+from django.core.management.base import BaseCommand
+
+from petale import models
+from petale.api_views import check_unknown_cuts
+
+
+def grouper(n, iterable, fillvalue=None):
+    "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
+    args = [iter(iterable)] * n
+    return itertools.zip_longest(fillvalue=fillvalue, *args)
+
+
+class Command(BaseCommand):
+    help = 'Clean petale data'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--delete', action='store_true', help='Delete dead petals.')
+
+    def handle(self, *args, delete=False, **options):
+        zombie_uuids = set()
+        for cuts in grouper(500, models.CUT.objects.iterator()):
+            zombie_uuids.update(check_unknown_cuts({cut.uuid for cut in cuts if cut is not None}))
+        if options['verbosity'] > 1:
+            print('Found %d dead cuts on %d known cuts.' % (len(zombie_uuids), models.CUT.objects.count()))
+        if delete:
+            count, count_by_models = models.CUT.objects.filter(uuid__in=zombie_uuids).delete()
+            if count and options['verbosity'] > 0:
+                print(
+                    'Deleted ',
+                    ', '.join(
+                        '%d %s' % (count, model.split('.')[-1]) for model, count in count_by_models.items()
+                    ),
+                )

petale/migrations/0006_auto_20171017_1625.py

@@ -11,12 +11,14 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='partner',
             name='hard_global_max_size',
-            field=models.IntegerField(help_text='as kilobytes', verbose_name='Hard max size'),
+            field=models.IntegerField(default=0, help_text='as kilobytes', verbose_name='Hard max size'),
         ),
         migrations.AlterField(
             model_name='partner',
             name='hard_per_key_max_size',
-            field=models.IntegerField(help_text='as kilobytes', verbose_name='Hard max size per key'),
+            field=models.IntegerField(
+                default=0, help_text='as kilobytes', verbose_name='Hard max size per key'
+            ),
         ),
         migrations.AlterField(
             model_name='partner',
@@ -26,11 +28,13 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='partner',
             name='soft_global_max_size',
-            field=models.IntegerField(help_text='as kilobytes', verbose_name='Soft max size'),
+            field=models.IntegerField(default=0, help_text='as kilobytes', verbose_name='Soft max size'),
         ),
         migrations.AlterField(
             model_name='partner',
             name='soft_per_key_max_size',
-            field=models.IntegerField(help_text='as kilobytes', verbose_name='Soft max size per key'),
+            field=models.IntegerField(
+                default=0, help_text='as kilobytes', verbose_name='Soft max size per key'
+            ),
         ),
     ]

petale/models.py

@@ -37,13 +37,17 @@ class Partner(models.Model):
         blank=True,
         help_text=_('List of admin emails separated by comma'),
     )
-    hard_global_max_size = models.IntegerField(verbose_name=_('Hard max size'), help_text=_('as kilobytes'))
-    soft_global_max_size = models.IntegerField(verbose_name=_('Soft max size'), help_text=_('as kilobytes'))
+    hard_global_max_size = models.IntegerField(
+        verbose_name=_('Hard max size'), default=0, help_text=_('as kilobytes')
+    )
+    soft_global_max_size = models.IntegerField(
+        verbose_name=_('Soft max size'), default=0, help_text=_('as kilobytes')
+    )
     hard_per_key_max_size = models.IntegerField(
-        verbose_name=_('Hard max size per key'), help_text=_('as kilobytes')
+        verbose_name=_('Hard max size per key'), default=0, help_text=_('as kilobytes')
     )
     soft_per_key_max_size = models.IntegerField(
-        verbose_name=_('Soft max size per key'), help_text=_('as kilobytes')
+        verbose_name=_('Soft max size per key'), default=0, help_text=_('as kilobytes')
     )
     size = models.BigIntegerField(verbose_name=_('Size'), default=0, help_text=_('as bytes'))
 

tests/test_commands.py

@@ -0,0 +1,69 @@
+from unittest import mock
+
+import pytest
+from django.core.files.base import ContentFile
+from django.core.management import call_command
+
+from petale.api_views import check_unknown_cuts
+from petale.models import CUT, Partner, Petal
+
+
+@pytest.fixture(autouse=True)
+def settings(settings):
+    settings.PETALE_AUTHENTIC_URL = 'https://authentic.example.com/'
+    settings.PETALE_AUTHENTIC_CREDS = 'admin:admin'
+
+
+@pytest.fixture
+def requests_post():
+    post_return_value = mock.Mock()
+    post_return_value.json.return_value = None
+    with mock.patch('requests.post', return_value=post_return_value) as requests_post:
+        yield requests_post
+
+
+def test_check_unknown_cuts(requests_post):
+
+    requests_post.return_value.json.return_value = {'unknown_uuids': ['1']}
+    assert check_unknown_cuts(['1']) == ['1']
+    assert requests_post.call_args[0] == ('https://authentic.example.com/api/users/synchronization/',)
+    assert requests_post.call_args[1] == {
+        'auth': 'admin:admin',
+        'json': {'known_uuids': ['1']},
+        'verify': False,
+    }
+
+
+class TestClean:
+
+    uuids = {str(i) for i in range(10)}
+    unknown_uuids = {str(i) for i in range(5)}
+
+    @pytest.fixture
+    def partner(self, db):
+        return Partner.objects.create(name='partner')
+
+    @pytest.fixture
+    def cuts(self, partner):
+        for uuid in self.uuids:
+            cut = CUT.objects.create(uuid=uuid)
+            petal = Petal.objects.create(name='petal', partner=partner, cut=cut, size=0)
+            petal.data.save('petal.dat', ContentFile(b'1234'))
+
+    @pytest.fixture
+    def requests_post(self, requests_post):
+        requests_post.return_value.json.return_value = {'unknown_uuids': self.unknown_uuids}
+        return requests_post
+
+    def test_dry(self, cuts, partner, requests_post):
+        call_command('clean')
+        assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids
+        assert set(CUT.objects.values_list('uuid', flat=True)) == self.uuids
+
+    def test_delete(self, cuts, partner, requests_post):
+        call_command('clean', '--delete')
+        assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids
+        assert set(CUT.objects.values_list('uuid', flat=True)) == self.uuids - self.unknown_uuids
+
+        call_command('clean', '--delete')
+        assert set(requests_post.call_args[1]['json']['known_uuids']) == self.uuids - self.unknown_uuids